Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_confirm_operations,
25        validate_hello_event_declarations, validate_hello_flow_operations,
26        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29    },
30    peer::{
31        encode_route_bind, ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY,
32        OPERATOR_APP_CAPABILITY,
33    },
34    scope::{
35        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
36        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
37        SCOPE_SYNC_OP,
38    },
39    session::{
40        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
41        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
42        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
43        ROLE_VERSIONS_FIELD,
44    },
45    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
46    Priority, RouteTarget, PROTOCOL_VERSION,
47};
48use tokio::time::{timeout_at, Instant};
49use tracing::{debug, info, warn};
50
51use crate::{
52    caller_identity::CallerIdentity,
53    capability_requirements::{
54        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
55        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
56        RequirementStatus, RuntimeModule,
57    },
58    child_roster::DaemonShutdownFlag,
59    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
60    forwarding::{
61        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
62        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
63        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
64    },
65    observability::{
66        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
67    },
68    privileged_prompt::{prompt_text, PrivilegedOperation},
69    provenance::{
70        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
71    },
72    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
73    router::{daemon_error_flags, RouteCtx, RouterError},
74    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
75    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
76    stderr_tail::{CaptureState, TailEntry},
77    supervise::{
78        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
79        SupervisorHandle, SwapHelloAdmission,
80    },
81    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
82};
83
84/// Lowest envelope version this subc build will negotiate.
85///
86/// Module HELLO negotiation is exact: peers must use the daemon's locked
87/// protocol version. Older and newer peers receive `version_unsupported` and
88/// are not registered.
89pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
90
91const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
92const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
93const CAP_PING_PONG: &str = "ping_pong_v1";
94const CAP_SESSION_ATTACH: &str = "session_attach_v1";
95const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
96
97const SUBC_CONTROL_OPS: &[&str] = &[
98    ops::ACTIVITY_RECORD,
99    ops::ACTIVITY_QUERY,
100    ops::ACTIVITY_SUBSCRIBE,
101    ops::SERVER_DESCRIBE,
102    ops::CATALOG_LIST,
103    ops::ROUTE_OPEN,
104    ops::ROUTE_POLL,
105    ops::ROUTE_CLOSING,
106    ops::ROUTE_CLOSED,
107    ops::SUPERVISOR_LIST,
108    ops::SUPERVISOR_RESTART,
109    ops::SUPERVISOR_SWAP,
110    ops::SUPERVISOR_RELOAD,
111    ops::SUPERVISOR_RESCAN,
112    ops::SUPERVISOR_RELEASE_RESERVED,
113    ops::SUPERVISOR_SET_ENABLED,
114    ops::SUPERVISOR_HEALTH_PROBE,
115    ops::SUPERVISOR_HEALTH,
116    ops::SUPERVISOR_STDERR_TAIL,
117    ops::SUPERVISOR_TERMINALS,
118    ops::SUPERVISOR_ROUTES,
119    ops::SUPERVISOR_PROVENANCE,
120    ops::SUPERVISOR_SPAWN_SNAPSHOT,
121    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
122];
123
124/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
125/// announced by a capability in `server.describe` instead, and clients check
126/// that capability before sending it. `ck daemon --json` prints `subc_ops`
127/// verbatim and scripts read that output, so a new name there would change it.
128/// A malformed body for one of these is still refused `invalid_control_body`
129/// rather than as an unknown op.
130const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
131
132mod rescan_checked;
133use rescan_checked::{RescanMode, RescanPlan};
134
135const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
136    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
137    "supervisor.live_roots",
138    SCOPE_SYNC_OP,
139    SCOPE_APPLY_OP,
140    SCOPE_DESCRIBE_OP,
141    "operator.confirm",
142];
143
144/// Module-originated ops the daemon answers but does not advertise in
145/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
146/// belongs to is incomplete, so no module is told it works before it does.
147const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
148
149const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
150
151/// How long subc waits for a module to ack a relayed route.bind before returning
152/// `module_timeout`. The ack waits on the module's own configure, which for AFT
153/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
154/// and DB-open work — on a cold page cache or a large repo that legitimately
155/// exceeds a couple of seconds. The default is generous because rejecting a VALID
156/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
157/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
158pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
159
160/// How many CONSECUTIVE full-budget relay timeouts against one target module
161/// open that module's bind-relay breaker.
162///
163/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
164/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
165/// so three consecutive full-budget timeouts take ~36s to observe: every client
166/// whose open contributed to opening the breaker had already given up on its
167/// own. That is what makes opening the breaker unable to turn a call that would
168/// have succeeded into a refusal — it can only make an already-failing module
169/// fail faster.
170///
171/// Two would be reachable inside one default deadline. One would convict a
172/// module on a single cold-cache bind, which is exactly the valid-but-slow case
173/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
174pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
175
176/// How long a module's bind-relay breaker stays open before exactly one
177/// `route.open` is let through as a probe.
178///
179/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
180/// re-pays a full-budget stall almost continuously, and the breaker stops being
181/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
182/// deadline: a client that starts retrying after the module recovers has to get
183/// a probe opportunity inside its own deadline, or the breaker converts a
184/// recovered module into a failed call — the failure it exists to prevent,
185/// pointed the other way.
186///
187/// 20s sits between those with room on both sides, and it caps what a wedged
188/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
189/// rather than one per `route.open` per connection. The stall that motivated
190/// this, with its measurements, is written up in
191/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
192/// waited the whole budget out.
193pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
194
195const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
196const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
197
198fn reload_verdict(
199    configured: &Path,
200    spawned_from: Option<&Path>,
201    image: subc_control::RunningImageAgreement,
202) -> PendingReloadVerdict {
203    let path = match spawned_from {
204        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
205            ReloadPathAgreement::Match
206        }
207        Some(spawned_from) => ReloadPathAgreement::Mismatch {
208            configured: configured.to_path_buf(),
209            spawned_from: spawned_from.to_path_buf(),
210        },
211        None => ReloadPathAgreement::Unavailable {
212            reason: if matches!(
213                image,
214                subc_control::RunningImageAgreement::Unavailable {
215                    reason: subc_control::RunningImageUnavailableReason::NotRunning
216                }
217            ) {
218                ReloadPathUnavailableReason::NotRunning
219            } else {
220                ReloadPathUnavailableReason::SpawnedPathUnavailable
221            },
222        },
223    };
224    PendingReloadVerdict {
225        path,
226        image,
227        spec: None,
228    }
229}
230
231fn launch_spec_agreement(
232    configured: &crate::supervise::ModuleSpec,
233    running: bool,
234    spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237    if !running {
238        return Agreement::Unavailable {
239            reason: Reason::NotRunning,
240        };
241    }
242    match spawned {
243        None => Agreement::Unavailable {
244            reason: Reason::SpawnedSpecUnavailable,
245        },
246        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247            Agreement::Match
248        }
249        Some(_) => Agreement::Mismatch,
250    }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254    if configured == spawned_from {
255        return true;
256    }
257    #[cfg(windows)]
258    {
259        // Windows spawn records the resolved verbatim path (\\?\...), while
260        // configuration keeps its original spelling. Resolve both with the OS
261        // instead of stripping prefixes or folding case, which could conflate
262        // distinct names in case-sensitive directories. Different resolved
263        // paths still differ even if their files have identical content.
264        match (
265            std::fs::canonicalize(configured),
266            std::fs::canonicalize(spawned_from),
267        ) {
268            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269            _ => false,
270        }
271    }
272    #[cfg(not(windows))]
273    {
274        false
275    }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280    build: DaemonBuildProvenance,
281    pid: Option<u32>,
282    started_at_ms: Option<u64>,
283    start_clock: Option<crate::clock::StartClock>,
284    executable_path: Option<PathBuf>,
285    executable_identity: Option<SpawnedFileIdentity>,
286    process_start_time: Option<u64>,
287    probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291    fn default() -> Self {
292        Self {
293            build: DaemonBuildProvenance {
294                build_git_sha: None,
295                build_lock_digest: None,
296            },
297            pid: None,
298            started_at_ms: None,
299            start_clock: None,
300            executable_path: None,
301            executable_identity: None,
302            process_start_time: None,
303            probe: ExecutableIdentityProbe::default(),
304        }
305    }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310    supervisor: Supervisor,
311    config_path: PathBuf,
312    configured_port: Option<u16>,
313    storage_config: Option<crate::daemon_config::StorageConfig>,
314    admission_facts_carrier_module_id: Option<String>,
315    admission_facts_targets: Option<Vec<String>>,
316    scope_authority_owners: Vec<String>,
317    privileged_operator_prompt: PrivilegedOperatorPrompt,
318    /// Counts executed rescans that set out to change the supervised modules'
319    /// staged or applied configuration, counted before the first change so a
320    /// rescan that fails partway still counts. With the daemon incarnation it
321    /// names the state a `supervisor.rescan_checked` preview was computed
322    /// against.
323    generation: Arc<std::sync::atomic::AtomicU64>,
324}
325
326/// Refusal labels passed to `observe_route_open_refusal` that mean the target
327/// module is not serving right now, and so open or extend an outage in the
328/// route outage tracker. Every one of them is only reachable after the target
329/// was found in the registry, which is what keeps an arbitrary client-chosen
330/// id from ever creating tracker state.
331///
332/// Deliberately absent: `not_registered` and `removed` (the id may be
333/// anything a client sent, and a removed module is gone on purpose),
334/// `protocol_none` (such a module never serves routes, so nothing is out),
335/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
336/// capability and admission-facts refusals (they refuse the caller, not a
337/// module outage), and `relay_reservation_failed` (its code ranges over
338/// capacity limits as well as a vanished connection). Capacity, breaker,
339/// relay-timeout and module-rejection refusals do not pass through that
340/// function at all; the breaker logs its own transitions.
341///
342/// The two not-serving refusals that bypass that function record themselves
343/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
344/// `required_capability_unprovided` is not tracked: the module itself is up,
345/// and the outage belongs to the missing provider.
346const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
347    "reloading",
348    "supervisor_not_live",
349    "registration_not_active",
350    "no_forwarding_connection",
351    "relay_send_failed",
352];
353
354/// Real channel-0 control handler for subc itself.
355#[derive(Clone)]
356pub struct ControlHandler {
357    registry: Arc<Registry>,
358    activity: Option<crate::activity::ActivityLedger>,
359    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
360    forwarding: Arc<ForwardingTable>,
361    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
362    supervisor: SupervisorHandle,
363    subc_capabilities: Arc<[String]>,
364    /// Daemon-wide route.bind relay budget. Used as the fallback when the
365    /// target module has no per-module override in
366    /// `route_bind_relay_timeouts`.
367    route_bind_relay_timeout: Duration,
368    /// Per-module route.bind relay budget overrides, keyed by module id. When
369    /// `handle_route_open` resolves the deadline for a target module, a
370    /// per-module entry wins over the daemon-wide value above.
371    route_bind_relay_timeouts: BTreeMap<String, Duration>,
372    /// Per-target-module bind-relay breaker state. Shared with the forwarding
373    /// table, which is where a new module connection resets it.
374    route_bind_breakers: RouteBindBreakers,
375    /// Live relay admissions keyed by target module. Shared through the
376    /// forwarding table so cloned or separately built handlers enforce one cap.
377    route_bind_concurrency: RouteBindConcurrency,
378    /// Start and end of each module's not-serving period as seen by
379    /// `route.open`, so an outage gets one line at each edge instead of only
380    /// the per-refusal INFO lines. Taken from the forwarding table, so every
381    /// handler built over one table shares it.
382    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
383    /// Consecutive relay timeouts that open a module's breaker.
384    route_bind_breaker_threshold: u32,
385    /// How long a breaker stays open before one probe is admitted.
386    route_bind_breaker_cooldown: Duration,
387    health_probe_timeout: Duration,
388    /// Central storage policy. When set, each registering module receives its
389    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
390    storage_config: Option<crate::daemon_config::StorageConfig>,
391    /// The machine id established at boot, served on every HELLO_ACK and on
392    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
393    /// changes the file, never this value. `None` serves no id.
394    machine_id: Option<crate::machine_id::MachineId>,
395    admission_facts_carrier_module_id: Option<String>,
396    admission_facts_targets: Option<Vec<String>>,
397    /// Scope records with their sync authorities and tombstones; see
398    /// `crate::scopes`. Shared by clones of this handler, so every connection
399    /// reads and writes one table.
400    scopes: Arc<RwLock<ScopeTable>>,
401    /// Wakes the expiry loop when an accepted scope change may have added a
402    /// deadline, so the loop can wait without ticking while no live scope has
403    /// one. A permit is stored when nothing waits, so no wake is lost.
404    scope_deadline_added: Arc<tokio::sync::Notify>,
405    /// The configured `scope_authority_owners`, kept so a rescan can report a
406    /// changed value as needing a daemon restart; rescan never applies it.
407    scope_authority_owners: Vec<String>,
408    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
409    privileged_operator_prompt: PrivilegedOperatorPrompt,
410    /// The local and peer address of each accepted TCP client connection, read
411    /// from the daemon's own socket, so a privileged request can name its caller.
412    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
413    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
414    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
415    operator_app_policy: crate::operator_apps::OperatorAppPolicy,
416    operator_apps: Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417    #[cfg(test)]
418    caller_lookup: Option<TestCallerLookup>,
419    /// The launch nonce each module connection presented at HELLO, which is how
420    /// a `scope.sync` is matched to the owner's current launch.
421    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
422    rescan: Option<SupervisorRescanContext>,
423    connected_clients: ConnectedClients,
424    counters: DaemonCounters,
425    /// Why this daemon serves no local endpoint, when binding one failed. `None`
426    /// when the endpoint is served, and also on a handler that was never told.
427    local_endpoint_unavailable: Option<String>,
428    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
429    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
430    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
431    /// their usual severity (see [`log_requirement_events`]). A handler built
432    /// without one has a flag that is never set.
433    daemon_shutdown: DaemonShutdownFlag,
434    daemon_provenance: DaemonProvenanceFacts,
435    #[cfg(test)]
436    control_dispatch_delay: Option<Duration>,
437    #[cfg(test)]
438    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
439    /// Holds `route.open` after the route.bind is queued to the module and
440    /// before it is marked enqueued, so a test can make the module's reply (or
441    /// a client teardown) land exactly inside that window.
442    #[cfg(test)]
443    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
444    /// Holds a HELLO after its registry entry is reserved and before its
445    /// forwarding endpoint is installed, so a test can read the catalog and
446    /// open routes while the module is registered but not yet routable. A
447    /// blocking barrier because the HELLO handler is synchronous; the test
448    /// runs the HELLO on its own thread.
449    #[cfg(test)]
450    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
451}
452
453impl fmt::Debug for ControlHandler {
454    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
455        f.debug_struct("ControlHandler")
456            .field("registry", &self.registry)
457            .field("forwarding", &self.forwarding)
458            .field("process_liveness", &self.process_liveness.is_some())
459            .field("supervisor", &self.supervisor)
460            .field("subc_capabilities", &self.subc_capabilities)
461            .finish()
462    }
463}
464
465struct RouteOpenRequest {
466    target: RouteTarget,
467    identity: BindIdentity,
468    consumer_identity: Option<ConsumerIdentity>,
469    consumer_capabilities: Option<Vec<String>>,
470    role_versions: Option<BTreeMap<String, String>>,
471    admission_facts: Option<serde_json::Value>,
472    scope: Option<ScopeSelector>,
473}
474
475#[derive(Clone, Copy)]
476enum RouteOpenPhase {
477    Admission,
478    ProjectRoot,
479    EgressReservation,
480    BindSend,
481    BindAckWait,
482}
483
484/// Store elapsed time for each route-open phase, including attempts that end
485/// in an early refusal or backend error. Admission also includes rechecks and
486/// commit bookkeeping performed between the other timed phases.
487struct RouteOpenTimings {
488    module_id: String,
489    principal: String,
490    admission: Duration,
491    project_root: Duration,
492    egress_reservation: Duration,
493    bind_send: Duration,
494    bind_ack_wait: Duration,
495    phase: Option<RouteOpenPhase>,
496    phase_started_at: Instant,
497}
498
499impl RouteOpenTimings {
500    fn new() -> Self {
501        Self {
502            module_id: String::new(),
503            // Refusals before attestation must not claim a verified principal.
504            principal: "unresolved".to_string(),
505            admission: Duration::ZERO,
506            project_root: Duration::ZERO,
507            egress_reservation: Duration::ZERO,
508            bind_send: Duration::ZERO,
509            bind_ack_wait: Duration::ZERO,
510            phase: Some(RouteOpenPhase::Admission),
511            phase_started_at: Instant::now(),
512        }
513    }
514
515    fn finish(&mut self) {
516        let elapsed = self.phase_started_at.elapsed();
517        match self.phase.take() {
518            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
519            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
520            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
521            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
522            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
523            None => {}
524        }
525    }
526
527    fn enter(&mut self, phase: RouteOpenPhase) {
528        self.finish();
529        self.phase = Some(phase);
530        self.phase_started_at = Instant::now();
531    }
532}
533
534struct RouteBindReservationGuard {
535    forwarding: Arc<ForwardingTable>,
536    endpoint: ModuleEndpointId,
537    relay_corr: u64,
538    armed: bool,
539}
540
541struct ModuleControlRpcGuard {
542    forwarding: Arc<ForwardingTable>,
543    endpoint: ModuleEndpointId,
544    corr: u64,
545    armed: bool,
546}
547
548impl ModuleControlRpcGuard {
549    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
550        Self {
551            forwarding,
552            endpoint,
553            corr,
554            armed: true,
555        }
556    }
557
558    fn disarm(&mut self) {
559        self.armed = false;
560    }
561}
562
563impl Drop for ModuleControlRpcGuard {
564    fn drop(&mut self) {
565        if self.armed {
566            let _ = self
567                .forwarding
568                .cancel_module_control_rpc(self.endpoint, self.corr);
569        }
570    }
571}
572
573impl RouteBindReservationGuard {
574    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
575        Self {
576            forwarding,
577            endpoint,
578            relay_corr,
579            armed: true,
580        }
581    }
582
583    fn release_and_disarm(&mut self) {
584        if !self.armed {
585            return;
586        }
587        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
588            self.endpoint,
589            self.relay_corr,
590            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
591        ) {
592            send_goodbye_target_best_effort(
593                &self.forwarding.counters(),
594                &target,
595                "canceled route.bind",
596            );
597        }
598        self.armed = false;
599    }
600
601    fn disarm(&mut self) {
602        self.armed = false;
603    }
604}
605
606impl Drop for RouteBindReservationGuard {
607    fn drop(&mut self) {
608        self.release_and_disarm();
609    }
610}
611
612/// Per-target-module circuit breaker around the `route.bind` relay.
613///
614/// The connection reader is serial per connection, so a module whose `on_bind`
615/// sits on the ack blocks every LATER frame on the connections that call it,
616/// including calls to unrelated modules. This does not make any module's bind
617/// fast; it stops the daemon paying the full budget again and again for a
618/// condition it has already observed.
619///
620/// State is keyed by TARGET MODULE and shared by every connection: a wedged
621/// module wedges everyone, so what one connection learned should protect the
622/// rest.
623///
624/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
625/// relay to that module has actually timed out, and is removed again when a
626/// relay is accepted or the module reconnects, so it cannot grow with traffic
627/// or with modules that behave.
628///
629/// # Why a `std` mutex here is not the head-of-line defect again
630///
631/// Acquisition never awaits. The critical section is a hash lookup plus a few
632/// integer updates, with no I/O and no `.await` inside it, so a reader task
633/// cannot be descheduled behind it the way it can behind
634/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
635/// primitive, held for the same kind of work, as the refusal counter this very
636/// path already increments.
637///
638/// It is also NOT on the data-plane splice path: only `route.open` and module
639/// registration touch it, so bound-route frames gain no state check and no
640/// contention.
641#[derive(Debug, Clone, Default)]
642pub(crate) struct RouteBindBreakers {
643    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
644}
645
646#[derive(Debug, Clone, Default)]
647pub(crate) struct RouteBindConcurrency {
648    modules: Arc<Mutex<HashMap<String, usize>>>,
649}
650
651struct RouteBindConcurrencyGuard {
652    concurrency: RouteBindConcurrency,
653    module_id: String,
654}
655
656impl RouteBindConcurrency {
657    /// Admit without waiting. Waiting here would move the bind stall from the
658    /// module reply to a semaphore and restore reader head-of-line blocking.
659    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
660        let mut modules = self
661            .modules
662            .lock()
663            .expect("route.bind concurrency mutex poisoned");
664        let in_flight = modules.entry(module_id.to_string()).or_default();
665        if *in_flight >= limit {
666            return Err(*in_flight);
667        }
668        *in_flight += 1;
669        Ok(RouteBindConcurrencyGuard {
670            concurrency: self.clone(),
671            module_id: module_id.to_string(),
672        })
673    }
674}
675
676impl Drop for RouteBindConcurrencyGuard {
677    fn drop(&mut self) {
678        let mut modules = self
679            .concurrency
680            .modules
681            .lock()
682            .expect("route.bind concurrency mutex poisoned");
683        let remove = {
684            let in_flight = modules
685                .get_mut(&self.module_id)
686                .expect("admitted route.bind has a concurrency entry");
687            *in_flight -= 1;
688            *in_flight == 0
689        };
690        if remove {
691            modules.remove(&self.module_id);
692        }
693    }
694}
695
696#[derive(Debug, Default)]
697struct ModuleBreakerState {
698    /// Relay timeouts observed with no accepted relay in between.
699    consecutive_timeouts: u32,
700    /// `Some` while the breaker is open: the instant the cooldown expires and
701    /// the next arrival may probe. `None` means closed.
702    cooldown_until: Option<Instant>,
703    /// A half-open probe has been admitted and has not settled yet. This is
704    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
705    /// that read the cooldown, so concurrent opens arriving at the moment the
706    /// cooldown expires cannot all decide that they are the probe.
707    probe_in_flight: Option<Arc<()>>,
708}
709
710/// What the breaker decided for one `route.open`, before any relay work.
711enum RouteBindAdmission<'a> {
712    Admitted {
713        guard: RouteBindBreakerGuard<'a>,
714        /// This open is the single half-open probe, so the transition is worth
715        /// one log line.
716        probe: bool,
717    },
718    Refused {
719        consecutive_timeouts: u32,
720        /// What is left of the cooldown. Zero when the refusal is because the
721        /// one probe is already in flight rather than because the cooldown has
722        /// not elapsed.
723        retry_in: Duration,
724        probe_in_flight: bool,
725    },
726}
727
728/// An outstanding admission, which must be told how its relay settled.
729///
730/// `Drop` settles it as inconclusive, so an early return between admission and
731/// the relay -- or the whole handler being cancelled when the client
732/// disconnects -- releases a half-open probe slot instead of leaving the
733/// breaker wedged half-open with no further probes.
734struct RouteBindBreakerGuard<'a> {
735    breakers: RouteBindBreakers,
736    module_id: &'a str,
737    probe_token: Option<Arc<()>>,
738    settled: bool,
739}
740
741impl RouteBindBreakerGuard<'_> {
742    /// The module answered within the budget and took the bind. THE ONLY
743    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
744    /// breaker, which is a transition worth logging.
745    fn record_accepted(&mut self) -> bool {
746        self.settled = true;
747        self.breakers.record_accepted(self.module_id)
748    }
749
750    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
751    /// COUNTS TOWARD OPENING.
752    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
753        self.settled = true;
754        self.breakers.record_timeout(
755            self.module_id,
756            self.probe_token.as_ref(),
757            threshold,
758            cooldown,
759        )
760    }
761
762    /// Everything else: the module REJECTED the bind, its connection went away
763    /// mid-relay, or the waiter was cancelled.
764    ///
765    /// None of these is evidence that a module is slow, and each already has
766    /// its own refusal with its own code. A module that rejects a bind in
767    /// microseconds is healthy and must never be convicted for it; a module
768    /// that died has said nothing about the module that replaces it. So these
769    /// neither increment nor reset the count -- they only release a probe slot.
770    fn record_inconclusive(&mut self) {
771        self.settled = true;
772        self.breakers
773            .record_inconclusive(self.module_id, self.probe_token.as_ref());
774    }
775}
776
777impl Drop for RouteBindBreakerGuard<'_> {
778    fn drop(&mut self) {
779        if !self.settled {
780            self.breakers
781                .record_inconclusive(self.module_id, self.probe_token.as_ref());
782        }
783    }
784}
785
786/// The breaker moved to open, reported so the caller can log it outside the
787/// lock. Opening is rare and load-bearing; the refusals that follow are
788/// frequent and are counted rather than logged.
789struct BreakerOpened {
790    consecutive_timeouts: u32,
791    /// True when a failed probe re-opened an already-open breaker, which reads
792    /// very differently in a log from a first opening.
793    reopened_after_probe: bool,
794}
795
796impl RouteBindBreakers {
797    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
798        self.modules
799            .lock()
800            .expect("route.bind breaker mutex poisoned")
801    }
802
803    /// Decide whether this `route.open` may attempt its relay. Takes the map
804    /// lock and nothing else, and never awaits.
805    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
806        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
807            probe: probe_token.is_some(),
808            guard: RouteBindBreakerGuard {
809                breakers: self.clone(),
810                module_id,
811                probe_token,
812                settled: false,
813            },
814        };
815
816        let mut modules = self.lock();
817        let Some(state) = modules.get_mut(module_id) else {
818            return admitted(None);
819        };
820        let Some(cooldown_until) = state.cooldown_until else {
821            return admitted(None);
822        };
823        if state.probe_in_flight.is_some() {
824            return RouteBindAdmission::Refused {
825                consecutive_timeouts: state.consecutive_timeouts,
826                retry_in: Duration::ZERO,
827                probe_in_flight: true,
828            };
829        }
830        let now = Instant::now();
831        if now < cooldown_until {
832            return RouteBindAdmission::Refused {
833                consecutive_timeouts: state.consecutive_timeouts,
834                retry_in: cooldown_until - now,
835                probe_in_flight: false,
836            };
837        }
838        let token = Arc::new(());
839        state.probe_in_flight = Some(Arc::clone(&token));
840        admitted(Some(token))
841    }
842
843    fn record_accepted(&self, module_id: &str) -> bool {
844        self.lock()
845            .remove(module_id)
846            .is_some_and(|state| state.cooldown_until.is_some())
847    }
848
849    fn record_timeout(
850        &self,
851        module_id: &str,
852        probe_token: Option<&Arc<()>>,
853        threshold: u32,
854        cooldown: Duration,
855    ) -> Option<BreakerOpened> {
856        let mut modules = self.lock();
857        let state = modules.entry(module_id.to_string()).or_default();
858        let was_open = state.cooldown_until.is_some();
859        let was_probe = Self::owns_probe(state, probe_token);
860        if was_probe {
861            state.probe_in_flight = None;
862        }
863        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
864        if state.consecutive_timeouts < threshold {
865            return None;
866        }
867        state.cooldown_until = Some(Instant::now() + cooldown);
868        Some(BreakerOpened {
869            consecutive_timeouts: state.consecutive_timeouts,
870            reopened_after_probe: was_open && was_probe,
871        })
872    }
873
874    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
875        // A relay can finish after the breaker was reset or after it opened
876        // again and started a new probe. Only the guard whose token matches the
877        // active probe may release it, so a late relay never frees a newer probe.
878        state
879            .probe_in_flight
880            .as_ref()
881            .zip(token)
882            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
883    }
884
885    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
886        if let Some(state) = self.lock().get_mut(module_id) {
887            if Self::owns_probe(state, probe_token) {
888                state.probe_in_flight = None;
889            }
890        }
891    }
892
893    /// Discard what was learned about a module, because the process it was
894    /// learned about is gone. Returns the discarded count when it was non-zero.
895    ///
896    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
897    /// `module_id` is a configuration identity that outlives any particular
898    /// child; what the breaker observed was the process behind the module
899    /// connection of the moment. When a new connection registers under that id
900    /// the verdict's subject no longer exists, so the verdict is stale by
901    /// construction rather than merely likely to be wrong. Keeping it would
902    /// apply a dead process's record to a live one, which is the same defect
903    /// class this breaker exists to stop the daemon committing.
904    ///
905    /// A half-open probe in flight is discarded with the rest: it was a
906    /// question about the old process.
907    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
908        self.lock()
909            .remove(module_id)
910            .map(|state| state.consecutive_timeouts)
911            .filter(|discarded| *discarded > 0)
912    }
913
914    /// Open breakers, for the `server.describe` counters object. `None` when
915    /// none is open, so the key stays absent rather than present-and-empty.
916    ///
917    /// This is the operator's answer to "is this module refusing instantly or
918    /// is it fine?", which look identical from a client that retries and then
919    /// succeeds.
920    fn open_snapshot(&self) -> Option<serde_json::Value> {
921        let now = Instant::now();
922        let modules = self.lock();
923        let open = modules
924            .iter()
925            .filter_map(|(module_id, state)| {
926                let cooldown_until = state.cooldown_until?;
927                Some((
928                    module_id.clone(),
929                    serde_json::json!({
930                        "consecutive_timeouts": state.consecutive_timeouts,
931                        "cooldown_remaining_ms":
932                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
933                        "probe_in_flight": state.probe_in_flight.is_some(),
934                    }),
935                ))
936            })
937            .collect::<serde_json::Map<String, serde_json::Value>>();
938        (!open.is_empty()).then_some(serde_json::Value::Object(open))
939    }
940}
941
942impl ControlHandler {
943    pub fn new(registry: Arc<Registry>) -> Self {
944        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
945    }
946
947    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
948        let counters = forwarding.counters();
949        // Taken from the forwarding table rather than created here, so that the
950        // breaker a `route.open` consults is the same one a module's
951        // registration resets, however many handlers are built over one table.
952        let route_bind_breakers = forwarding.route_bind_breakers();
953        let route_bind_concurrency = forwarding.route_bind_concurrency();
954        let route_outages = forwarding.route_outages();
955        Self {
956            registry,
957            activity: None,
958            activity_streams: Arc::new(Mutex::new(HashMap::new())),
959            forwarding,
960            process_liveness: None,
961            supervisor: SupervisorHandle::new(),
962            subc_capabilities: Arc::from([
963                CAP_MANIFEST_REGISTRATION.to_string(),
964                CAP_CHANNEL_LIFECYCLE.to_string(),
965                CAP_PING_PONG.to_string(),
966                CAP_SESSION_ATTACH.to_string(),
967                CAP_ADMISSION_FACTS_RELAY.to_string(),
968                CAP_SCOPES_V1.to_string(),
969                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
970                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
971                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
972                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
973                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
974                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
975                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
976                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
977                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
978                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
979                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
980            ]),
981            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
982            route_bind_relay_timeouts: BTreeMap::new(),
983            route_bind_breakers,
984            route_bind_concurrency,
985            route_outages,
986            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
987            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
988            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
989            storage_config: None,
990            machine_id: None,
991            admission_facts_carrier_module_id: None,
992            admission_facts_targets: None,
993            scopes: Arc::new(RwLock::new(ScopeTable::new(
994                crate::daemon_config::default_scope_authority_owners(),
995            ))),
996            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
997            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
998            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
999            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1000            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1001            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1002            operator_app_policy: Default::default(),
1003            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1004            #[cfg(test)]
1005            caller_lookup: None,
1006            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1007            rescan: None,
1008            connected_clients: ConnectedClients::new(),
1009            local_endpoint_unavailable: None,
1010            counters,
1011            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1012            daemon_shutdown: DaemonShutdownFlag::default(),
1013            daemon_provenance: DaemonProvenanceFacts::default(),
1014            #[cfg(test)]
1015            control_dispatch_delay: None,
1016            #[cfg(test)]
1017            provenance_probe_override: None,
1018            #[cfg(test)]
1019            route_bind_relay_sent_hold: None,
1020            #[cfg(test)]
1021            hello_registry_to_forwarding_hold: None,
1022        }
1023    }
1024
1025    /// Set the central storage policy: registering modules then receive their
1026    /// resolved storage descriptor in HELLO_ACK.
1027    pub fn with_storage_config(
1028        mut self,
1029        storage_config: Option<crate::daemon_config::StorageConfig>,
1030    ) -> Self {
1031        self.storage_config = storage_config;
1032        self
1033    }
1034
1035    /// Open the ledger on its own writer thread; an unavailable sink does not
1036    /// prevent the daemon from serving routing or supervisor requests.
1037    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1038        self.activity = Some(crate::activity::ActivityLedger::open(
1039            run_dir.join("activity.jsonl"),
1040            incarnation,
1041        ));
1042        self.supervisor.set_activity(self.activity.clone());
1043        self
1044    }
1045
1046    /// Set the machine id served to every registering module (HELLO_ACK) and on
1047    /// `server.describe`.
1048    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1049        self.machine_id = machine_id;
1050        self
1051    }
1052
1053    /// Configure the exact reserved module and target ids permitted to relay
1054    /// opaque admission facts. Config-file loading validates this authority;
1055    /// this builder keeps the same policy available to embedded test daemons.
1056    pub fn with_admission_facts_config(
1057        mut self,
1058        carrier_module_id: Option<String>,
1059        targets: Option<Vec<String>>,
1060    ) -> Self {
1061        self.admission_facts_carrier_module_id = carrier_module_id;
1062        self.admission_facts_targets = targets;
1063        self
1064    }
1065
1066    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1067    /// Replaces the scope table with an empty one under the new list, so call it
1068    /// while building the handler, before any module can sync.
1069    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1070        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1071        self.scope_authority_owners = owners;
1072        self
1073    }
1074
1075    /// Set the startup policy before installing the rescan context or serving requests.
1076    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1077        self.privileged_operator_prompt = mode;
1078        self
1079    }
1080
1081    /// Override the route.bind relay timeout. Used by tests that assert the
1082    /// timeout path so they don't block on the production-safe default.
1083    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1084        self.route_bind_relay_timeout = timeout;
1085        self
1086    }
1087
1088    /// Install per-module route.bind relay budget overrides. A module id
1089    /// listed here wins over the daemon-wide default set via
1090    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1091    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1092    /// the same `Duration` the bind path will use.
1093    pub fn with_route_bind_relay_timeouts(
1094        mut self,
1095        timeouts: impl IntoIterator<Item = (String, Duration)>,
1096    ) -> Self {
1097        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1098        self
1099    }
1100
1101    /// Resolve the route.bind relay budget for a specific target module id.
1102    /// Per-module overrides win; the daemon-wide value (set via
1103    /// `with_route_bind_relay_timeout` or the built-in default) is the
1104    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1105    /// the same resolution `handle_route_open` will use.
1106    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1107        self.route_bind_relay_timeouts
1108            .get(module_id)
1109            .copied()
1110            .unwrap_or(self.route_bind_relay_timeout)
1111    }
1112
1113    /// Override the per-module bind-relay breaker policy.
1114    ///
1115    /// Used by tests, which cannot spend three production budgets opening a
1116    /// breaker or twenty seconds waiting for its cooldown. The production
1117    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1118    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1119    /// reasoning for the numbers.
1120    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1121        self.route_bind_breaker_threshold = threshold.max(1);
1122        self.route_bind_breaker_cooldown = cooldown;
1123        self
1124    }
1125
1126    #[cfg(test)]
1127    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1128        self.health_probe_timeout = timeout;
1129        self
1130    }
1131
1132    #[cfg(test)]
1133    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1134        self.control_dispatch_delay = Some(delay);
1135        self
1136    }
1137
1138    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1139    /// announce it is parked, once to be released. Build the barrier for two
1140    /// parties, the test being the other one.
1141    #[cfg(test)]
1142    pub(crate) fn with_route_bind_relay_sent_hold(
1143        mut self,
1144        hold: Arc<tokio::sync::Barrier>,
1145    ) -> Self {
1146        self.route_bind_relay_sent_hold = Some(hold);
1147        self
1148    }
1149
1150    /// A HELLO waits on `hold` twice between reserving its registry entry and
1151    /// installing its forwarding endpoint: once to announce it is parked, once
1152    /// to be released. Build the barrier for two parties, the test being the
1153    /// other one, and run the HELLO on a thread of its own.
1154    #[cfg(test)]
1155    pub(crate) fn with_hello_registry_to_forwarding_hold(
1156        mut self,
1157        hold: Arc<std::sync::Barrier>,
1158    ) -> Self {
1159        self.hello_registry_to_forwarding_hold = Some(hold);
1160        self
1161    }
1162
1163    #[cfg(test)]
1164    fn hold_between_registry_and_forwarding(&self) {
1165        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1166            hold.wait();
1167            hold.wait();
1168        }
1169    }
1170
1171    pub fn with_process_liveness(
1172        mut self,
1173        process_liveness: Arc<dyn ModuleProcessLiveness>,
1174    ) -> Self {
1175        self.process_liveness = Some(process_liveness);
1176        self
1177    }
1178
1179    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1180        self.supervisor = supervisor;
1181        self.supervisor.set_activity(self.activity.clone());
1182        self
1183    }
1184
1185    pub fn with_daemon_provenance(
1186        mut self,
1187        pid: u32,
1188        started_at_ms: u64,
1189        executable_path: Option<PathBuf>,
1190        build_git_sha: Option<String>,
1191        build_lock_digest: Option<String>,
1192    ) -> Self {
1193        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1194        let process_start_time = process_start_time(pid);
1195        self.daemon_provenance = DaemonProvenanceFacts {
1196            build: DaemonBuildProvenance {
1197                build_git_sha,
1198                build_lock_digest,
1199            },
1200            pid: Some(pid),
1201            started_at_ms: Some(started_at_ms),
1202            start_clock: None,
1203            executable_path,
1204            executable_identity,
1205            process_start_time,
1206            probe: ExecutableIdentityProbe::default(),
1207        };
1208        self
1209    }
1210
1211    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1212        self.daemon_provenance.start_clock = Some(clock);
1213        self
1214    }
1215
1216    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1217    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1218        self.scopes
1219            .write()
1220            .unwrap_or_else(|poisoned| poisoned.into_inner())
1221            .set_wall_clock(Arc::new(clock));
1222        self
1223    }
1224
1225    #[cfg(test)]
1226    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1227        self.provenance_probe_override = Some(result);
1228        self
1229    }
1230
1231    /// Install the configured module set and its reserved capability bindings.
1232    /// Bindings are configuration-scoped and may point at a provider that has not
1233    /// been installed yet, so this does not require the bound module to exist.
1234    pub fn with_capability_config(
1235        self,
1236        modules: impl IntoIterator<Item = (String, bool)>,
1237        reserved_capabilities: BTreeMap<String, String>,
1238    ) -> Self {
1239        self.capability_evaluator
1240            .configure(modules, reserved_capabilities);
1241        self
1242    }
1243
1244    /// Share the supervisor's daemon-shutdown flag with this handler, so
1245    /// requirement changes after the daemon has begun its announced shutdown
1246    /// are not reported as faults.
1247    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1248        self.daemon_shutdown = flag;
1249        self
1250    }
1251
1252    pub fn with_supervisor_rescan(
1253        mut self,
1254        supervisor: Supervisor,
1255        config_path: impl Into<PathBuf>,
1256        configured_port: Option<u16>,
1257    ) -> Self {
1258        self.rescan = Some(SupervisorRescanContext {
1259            supervisor,
1260            config_path: config_path.into(),
1261            configured_port,
1262            storage_config: self.storage_config.clone(),
1263            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1264            admission_facts_targets: self.admission_facts_targets.clone(),
1265            scope_authority_owners: self.scope_authority_owners.clone(),
1266            privileged_operator_prompt: self.privileged_operator_prompt,
1267            generation: Arc::default(),
1268        });
1269        self
1270    }
1271
1272    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1273        self.connected_clients = connected_clients;
1274        self
1275    }
1276
1277    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1278    /// an operator can see a bind failure without reading the daemon log.
1279    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1280        self.local_endpoint_unavailable = reason;
1281        self
1282    }
1283
1284    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1285        Arc::clone(&self.forwarding)
1286    }
1287
1288    pub(crate) fn counters(&self) -> DaemonCounters {
1289        self.counters.clone()
1290    }
1291
1292    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1293    /// requirement event without depending on an operator polling a status command.
1294    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1295        tokio::spawn(async move {
1296            loop {
1297                self.capability_evaluator
1298                    .wait_for_change_or_deadline()
1299                    .await;
1300                self.refresh_capability_requirements();
1301            }
1302        });
1303    }
1304
1305    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1306    /// deadline passed during sleep is enforced on the first tick after wake.
1307    /// While no live scope carries a deadline the loop waits for an accepted
1308    /// scope change instead of ticking, so a daemon without deadlines never
1309    /// wakes for expiry.
1310    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1311        tokio::spawn(async move {
1312            loop {
1313                let has_deadlines = self
1314                    .scopes
1315                    .read()
1316                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1317                    .has_deadlines();
1318                if !has_deadlines {
1319                    self.scope_deadline_added.notified().await;
1320                    continue;
1321                }
1322                tokio::time::sleep(Duration::from_secs(1)).await;
1323                if let Err(error) = self.sweep_expired_scopes() {
1324                    warn!(%error, "scope expiry sweep failed");
1325                }
1326            }
1327        });
1328    }
1329
1330    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1331        let mut table = self
1332            .scopes
1333            .write()
1334            .unwrap_or_else(|poisoned| poisoned.into_inner());
1335        let swept = table.sweep_expired();
1336        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1337        drop(table);
1338        self.close_scope_drained_routes(drained);
1339        Ok(())
1340    }
1341
1342    fn publish_scope_changes(
1343        &self,
1344        changes: &[crate::scopes::ScopeTagChange],
1345        expired: &[crate::scopes::ScopeExpired],
1346    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1347        if changes.is_empty() {
1348            return Ok(Vec::new());
1349        }
1350        let drained = self
1351            .forwarding
1352            .publish_scope_changes(changes)
1353            .map_err(RouterError::Forwarding)?;
1354        let mut counts = HashMap::new();
1355        for route in &drained {
1356            *counts
1357                .entry((
1358                    route.scope.owner.as_str(),
1359                    route.scope.scope_ref.as_str(),
1360                    route.scope.tag.scope_epoch,
1361                ))
1362                .or_insert(0usize) += 1;
1363        }
1364        for scope in expired {
1365            let routes_closed = counts
1366                .get(&(
1367                    scope.owner.as_str(),
1368                    scope.scope_ref.as_str(),
1369                    scope.scope_epoch,
1370                ))
1371                .copied()
1372                .unwrap_or(0);
1373            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1374                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1375                routes_closed, "scope expired");
1376        }
1377        Ok(drained)
1378    }
1379
1380    fn runtime_capability_snapshot(
1381        &self,
1382    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1383        let runtime = self
1384            .supervisor
1385            .list()
1386            .into_iter()
1387            .map(|module| {
1388                let status = module.status().map_err(|err| {
1389                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1390                })?;
1391                Ok(RuntimeModule {
1392                    module_id: status.module_id,
1393                    state: status.state,
1394                    enabled: status.enabled,
1395                })
1396            })
1397            .collect::<Result<Vec<_>, RouterError>>()?;
1398        // Include every registration, published or not. The capability
1399        // evaluator counts a module as providing its capabilities as soon as it
1400        // has registered, even before `catalog.list` shows it, so a HELLO in
1401        // progress does not change other modules' required-capability
1402        // verdicts. See `Registry::list_registered_modules`.
1403        let registrations = self.registry.list_registered_modules().map_err(|err| {
1404            RouterError::backend(
1405                0,
1406                0,
1407                format!("failed to list capability registrations: {err}"),
1408            )
1409        })?;
1410        let registrations = registrations
1411            .into_iter()
1412            .map(|registration| RegisteredModule {
1413                module_id: registration.manifest.module_id,
1414                module_version: registration.manifest.module_version,
1415                capabilities: registration.manifest.capabilities,
1416            })
1417            .collect();
1418        Ok((runtime, registrations))
1419    }
1420
1421    /// The capability side effects of a module becoming the active registration
1422    /// for its id: cache its manifest (logging how it differs from the cached
1423    /// one), run the deny census when its declarations call for one, and recompute the
1424    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1425    /// candidate's does not, and the supervisor does it at promotion instead,
1426    /// through [`crate::supervise::SwapPromotionObserver`].
1427    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1428        let cached_registration = RegisteredModule {
1429            module_id: registration.manifest.module_id.clone(),
1430            module_version: registration.manifest.module_version.clone(),
1431            capabilities: registration.manifest.capabilities.clone(),
1432        };
1433        let change = self.capability_evaluator.record_hello(&cached_registration);
1434        log_manifest_change(&cached_registration.module_id, &change);
1435        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1436            self.enforce_capability_denies();
1437        }
1438        self.refresh_capability_requirements();
1439    }
1440
1441    /// Point the shared supervisor handle at this handler for swap promotions.
1442    /// Called wherever a handler is put behind the `Arc` the router serves, so
1443    /// it can be held weakly.
1444    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1445        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1446            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1447        self.supervisor.set_swap_promotion_observer(observer);
1448    }
1449
1450    pub fn refresh_capability_requirements(&self) {
1451        match self.runtime_capability_snapshot() {
1452            Ok((runtime, registrations)) => {
1453                log_requirement_events(
1454                    self.capability_evaluator
1455                        .evaluate_now(&runtime, &registrations),
1456                    &self.daemon_shutdown,
1457                );
1458            }
1459            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1460        }
1461    }
1462
1463    /// Reconcile only live, attested route bindings after a capability deny edge
1464    /// or target claim was added. This is deliberately a control-plane census:
1465    /// the opaque forwarding hot path must not grow a per-frame capability check.
1466    fn enforce_capability_denies(&self) {
1467        let registrations = match self.registry.list_registered_modules() {
1468            Ok(snapshot) => snapshot,
1469            Err(err) => {
1470                warn!(error = %err, "failed to read registrations for capability deny census");
1471                return;
1472            }
1473        };
1474        let manifests = registrations
1475            .into_iter()
1476            .map(|registration| {
1477                (
1478                    registration.manifest.module_id.clone(),
1479                    registration.manifest,
1480                )
1481            })
1482            .collect::<BTreeMap<_, _>>();
1483        let census = match self.forwarding.route_census(None) {
1484            Ok(census) => census,
1485            Err(err) => {
1486                warn!(error = %err, "failed to read route census for capability deny enforcement");
1487                return;
1488            }
1489        };
1490
1491        for (target_module_id, routes) in census {
1492            let Some(target_manifest) = manifests.get(&target_module_id) else {
1493                continue;
1494            };
1495            let mut closed_routes = Vec::new();
1496            let mut goodbyes = Vec::new();
1497            for route in routes {
1498                let Principal::Reserved {
1499                    module_id: opening_module_id,
1500                } = &route.principal
1501                else {
1502                    continue;
1503                };
1504                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1505                    continue;
1506                };
1507                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1508                    continue;
1509                };
1510
1511                match self.forwarding.release_client_route(
1512                    route.goodbye_target.connection_id,
1513                    route.goodbye_target.channel,
1514                    route.goodbye_target.epoch,
1515                ) {
1516                    Ok(RouteRelease::Removed(module_goodbye)) => {
1517                        warn!(
1518                            opening_module_id,
1519                            target_module_id,
1520                            capability,
1521                            "force-closing route because an attested capability deny edge now matches"
1522                        );
1523                        goodbyes.push(module_goodbye);
1524                        goodbyes.push(route.goodbye_target.clone());
1525                        closed_routes.push(route);
1526                    }
1527                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1528                    Err(err) => warn!(
1529                        opening_module_id,
1530                        target_module_id,
1531                        capability,
1532                        error = %err,
1533                        "failed to force-close capability-denied route"
1534                    ),
1535                }
1536            }
1537
1538            if closed_routes.is_empty() {
1539                continue;
1540            }
1541            send_route_control_pushes(
1542                &self.forwarding,
1543                closed_routes,
1544                ClientControlPush::RouteClosed {
1545                    module_id: target_module_id,
1546                    channels: Vec::new(),
1547                    reason: RouteCloseReason::CapabilityDenied,
1548                    drained: false,
1549                    abandoned: 0,
1550                    excluded_subscriptions: 0,
1551                    terminal: Some(false),
1552                },
1553            );
1554            // route.closed records the reason; the route GOODBYE ends the
1555            // client's handle and settles any calls still pending on it.
1556            self.emit_route_goodbyes(goodbyes);
1557        }
1558    }
1559
1560    /// Why a registered module is not accepting new route binds, or `None` when
1561    /// it is. This is the module's effective readiness: its declared readiness
1562    /// first, then every `need: required` capability it declares evaluating to
1563    /// `provided`. `route.open` and `catalog.list` both read it here so the
1564    /// catalog never reports a module routable that `route.open` would refuse.
1565    fn not_ready_reason(
1566        &self,
1567        registration: &crate::registry::ModuleRegistration,
1568    ) -> Option<NotReadyReason> {
1569        if !registration.ready {
1570            return Some(NotReadyReason {
1571                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1572                capability: None,
1573            });
1574        }
1575        self.first_unprovided_required_capability(registration)
1576            .map(|capability| NotReadyReason {
1577                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1578                capability: Some(capability),
1579            })
1580    }
1581
1582    /// The lexicographically first capability this registration declares
1583    /// `need: required` whose evaluator verdict is not `provided`.
1584    ///
1585    /// The verdicts are the capability evaluator's own; nothing here decides
1586    /// what "provided" means. The evaluator counts a capability provided as
1587    /// soon as a module claiming it has REGISTERED, not once that module is
1588    /// ready. That distinction is what keeps two modules that require each
1589    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1590    /// is ready", each would wait for the other to become ready first and
1591    /// neither ever would. Do not tighten it to readiness.
1592    ///
1593    /// A required capability with no verdict at all means this registration's
1594    /// HELLO or catalog.update landed after the last recompute; recompute once
1595    /// rather than let a missing verdict read as either answer. If it is still
1596    /// missing (the recompute itself failed) the capability counts as
1597    /// unprovided: the refusal is retryable, and routing a module whose
1598    /// required provider is unknown is the outcome this check exists to stop.
1599    fn first_unprovided_required_capability(
1600        &self,
1601        registration: &crate::registry::ModuleRegistration,
1602    ) -> Option<String> {
1603        let required = registration
1604            .manifest
1605            .capabilities
1606            .iter()
1607            .flat_map(|declarations| declarations.requires.iter())
1608            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1609            .map(|requirement| requirement.capability.as_str())
1610            .collect::<BTreeSet<_>>();
1611        if required.is_empty() {
1612            return None;
1613        }
1614        let module_id = registration.manifest.module_id.as_str();
1615        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1616        if required
1617            .iter()
1618            .any(|capability| verdict(capability).is_none())
1619        {
1620            self.refresh_capability_requirements();
1621        }
1622        required
1623            .into_iter()
1624            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1625            .map(str::to_string)
1626    }
1627
1628    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1629        self.capability_evaluator
1630            .statuses()
1631            .into_iter()
1632            .map(capability_requirement_status)
1633            .collect()
1634    }
1635
1636    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1637    /// registration-release watch. The signal is what the supervisor waits on
1638    /// before spawning a replacement, so it must only fire once forwarding
1639    /// teardown is also done (see [`Self::cleanup_connection`] /
1640    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1641    /// admitted but its forwarding endpoint could not be installed.
1642    fn deregister_connection(
1643        &self,
1644        connection_id: ConnectionId,
1645        reason: RegistrationEndReason,
1646    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1647        self.registry
1648            .deregister_connection_with_reason(connection_id, reason)
1649    }
1650
1651    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1652        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1653            return None;
1654        }
1655        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1656            parse_client_control_request(&frame.body)
1657        else {
1658            return None;
1659        };
1660        Some(target_module_id(&target).to_string())
1661    }
1662
1663    pub(crate) fn route_open_capacity_refusal(
1664        &self,
1665        ctx: &RouteCtx,
1666        frame: &Frame,
1667        target_module_id: &str,
1668        in_flight: usize,
1669        limit: usize,
1670    ) -> Result<Frame, RouterError> {
1671        // Sent before the request reaches `handle_route_open`, so the caller
1672        // fields come from the raw frame here.
1673        let caller = self.route_open_caller_from_frame(frame);
1674        self.route_open_admission_refusal_frame(
1675            ctx,
1676            &caller,
1677            frame,
1678            target_module_id,
1679            "open_admission_full",
1680            (in_flight, limit),
1681            format!(
1682                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1683            ),
1684        )
1685    }
1686
1687    fn route_open_target_capacity_refusal(
1688        &self,
1689        ctx: &RouteCtx,
1690        caller: &RouteOpenCaller,
1691        frame: &Frame,
1692        target_module_id: &str,
1693        in_flight: usize,
1694    ) -> Result<Frame, RouterError> {
1695        self.route_open_admission_refusal_frame(
1696            ctx,
1697            caller,
1698            frame,
1699            target_module_id,
1700            "target_binds_full",
1701            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1702            format!(
1703                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1704            ),
1705        )
1706    }
1707
1708    /// Admission pressure clears as existing binds settle, so its refusal must
1709    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1710    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1711    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1712    /// cannot currently reach its target; `module_timeout` would falsely claim
1713    /// that a wait expired. A new, cleaner code would be terminal to deployed
1714    /// clients, so it requires a client-tolerance rollout before daemon emission.
1715    #[allow(clippy::too_many_arguments)]
1716    fn route_open_admission_refusal_frame(
1717        &self,
1718        ctx: &RouteCtx,
1719        caller: &RouteOpenCaller,
1720        frame: &Frame,
1721        target_module_id: &str,
1722        reason: &'static str,
1723        (in_flight, limit): (usize, usize),
1724        message: impl Into<String>,
1725    ) -> Result<Frame, RouterError> {
1726        let code = error_codes::TARGET_UNAVAILABLE;
1727        self.counters.increment_route_open_refused(code);
1728        info!(
1729            target: "control",
1730            code,
1731            reason,
1732            module_id = ?target_module_id,
1733            connection_id = ctx.connection_id.get(),
1734            principal = %caller.principal,
1735            scope = %caller.scope,
1736            harness = %caller.harness,
1737            in_flight,
1738            limit,
1739            "route.open refused"
1740        );
1741        control_error_frame(frame, code, message.into())
1742    }
1743
1744    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1745    ///
1746    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1747    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1748    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1749    #[cfg(test)]
1750    pub fn handle_control(
1751        &self,
1752        connection_id: ConnectionId,
1753        frame: Frame,
1754    ) -> Result<Vec<Frame>, RouterError> {
1755        match frame.header.ty {
1756            FrameType::Ping => Ok(vec![pong(&frame)?]),
1757            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1758            FrameType::Goodbye => self.handle_goodbye(connection_id),
1759            ty => Ok(vec![control_error_frame(
1760                &frame,
1761                "unsupported_control_frame",
1762                format!("unsupported channel-0 frame {ty:?}"),
1763            )?]),
1764        }
1765    }
1766
1767    pub async fn handle_control_frame(
1768        &self,
1769        ctx: &RouteCtx,
1770        frame: Frame,
1771    ) -> Result<Vec<Frame>, RouterError> {
1772        self.handle_control_frame_timed(ctx, frame, None).await
1773    }
1774
1775    #[cfg(any(test, feature = "test-support"))]
1776    pub fn with_test_operator_app_policy(
1777        mut self,
1778        policy: crate::operator_apps::TestOperatorAppPolicy,
1779    ) -> Self {
1780        self.operator_app_policy = policy.into();
1781        self
1782    }
1783
1784    pub(crate) fn record_captured_peer(
1785        &self,
1786        connection: ConnectionId,
1787        peer: subc_os::local_peer::CapturedPeer,
1788    ) {
1789        self.record_observed_peer(connection, peer.observation().clone());
1790        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1791            self.operator_apps
1792                .lock()
1793                .unwrap_or_else(|p| p.into_inner())
1794                .insert(connection, stamp);
1795        }
1796    }
1797
1798    pub(crate) fn record_observed_peer(
1799        &self,
1800        connection: ConnectionId,
1801        peer: subc_os::local_peer::ObservedPeer,
1802    ) {
1803        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1804        let peer = match peer {
1805            OsPeer::Unobserved => ObservedPeer::unobserved(),
1806            OsPeer::Observed(peer) => {
1807                let observed = match peer.user {
1808                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1809                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1810                };
1811                observed
1812                    .with_exe(
1813                        peer.exe
1814                            .and_then(|path| path.into_os_string().into_string().ok()),
1815                    )
1816                    .with_signing_id(peer.signing_id)
1817            }
1818        };
1819        self.observed_peers
1820            .lock()
1821            .unwrap_or_else(|p| p.into_inner())
1822            .insert(connection, peer);
1823    }
1824
1825    pub(crate) fn record_caller_endpoints(
1826        &self,
1827        connection: ConnectionId,
1828        endpoints: Option<(SocketAddr, SocketAddr)>,
1829    ) {
1830        let mut callers = self
1831            .caller_endpoints
1832            .lock()
1833            .unwrap_or_else(|p| p.into_inner());
1834        callers.remove(&connection);
1835        if let Some(endpoints) = endpoints {
1836            callers.insert(connection, endpoints);
1837        }
1838    }
1839
1840    #[cfg(test)]
1841    pub(crate) fn with_caller_lookup_for_test(
1842        mut self,
1843        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1844    ) -> Self {
1845        self.caller_lookup = Some(Arc::new(lookup));
1846        self
1847    }
1848
1849    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1850        let endpoints = self
1851            .caller_endpoints
1852            .lock()
1853            .unwrap_or_else(|p| p.into_inner())
1854            .get(&connection)
1855            .copied();
1856        #[cfg(test)]
1857        let lookup = self.caller_lookup.clone();
1858        tokio::task::spawn_blocking(move || {
1859            #[cfg(test)]
1860            if let Some(lookup) = lookup {
1861                return lookup(endpoints);
1862            }
1863            endpoints
1864                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1865                .unwrap_or_default()
1866        })
1867        .await
1868        .unwrap_or_default()
1869    }
1870
1871    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1872        frame.header.channel == 0
1873            && frame.header.ty == FrameType::Request
1874            && self
1875                .forwarding
1876                .module_endpoint_for_connection(ctx.connection_id)
1877                .is_ok_and(|endpoint| endpoint.is_none())
1878            && parse_client_control_request(&frame.body)
1879                .ok()
1880                .is_some_and(|request| privileged_operation(&request).is_some())
1881    }
1882
1883    pub(crate) async fn handle_control_frame_timed(
1884        &self,
1885        ctx: &RouteCtx,
1886        frame: Frame,
1887        dispatch_started_at: Option<StdInstant>,
1888    ) -> Result<Vec<Frame>, RouterError> {
1889        match frame.header.ty {
1890            FrameType::Ping => Ok(vec![pong(&frame)?]),
1891            FrameType::Hello => {
1892                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1893            }
1894            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1895            FrameType::Cancel => {
1896                // A Cancel on channel 0 names a waiting operator.confirm or a
1897                // subscription. Successful cancellation has no ordinary reply.
1898                if self
1899                    .forwarding
1900                    .operator_confirms()
1901                    .cancel(ctx.connection_id, frame.header.corr)
1902                    || self
1903                        .supervisor
1904                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1905                    || self.cancel_activity_subscription(ctx, &frame)?
1906                {
1907                    Ok(Vec::new())
1908                } else {
1909                    Ok(vec![control_error_frame(
1910                        &frame,
1911                        "unknown_subscription",
1912                        "no subscription has this correlation id",
1913                    )?])
1914                }
1915            }
1916            FrameType::Request => {
1917                // This additive operation is not part of the existing exhaustive
1918                // module-control enum. Probe the op before decoding that enum.
1919                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1920                if op
1921                    .as_ref()
1922                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1923                {
1924                    return self.handle_activity_record(ctx, frame).await;
1925                }
1926                if op.as_ref().is_some_and(|probe| {
1927                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1928                }) {
1929                    return self.handle_activity_read(ctx, frame).await;
1930                }
1931                if op
1932                    .as_ref()
1933                    .is_some_and(|probe| probe.op == "operator.confirm")
1934                {
1935                    return self.handle_operator_confirm(ctx, frame);
1936                }
1937                if op
1938                    .as_ref()
1939                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1940                {
1941                    return self.handle_scope_served_by(ctx.connection_id, frame);
1942                }
1943                if self
1944                    .forwarding
1945                    .module_endpoint_for_connection(ctx.connection_id)
1946                    .map_err(RouterError::Forwarding)?
1947                    .is_some()
1948                {
1949                    if !is_known_module_request_op(&frame.body) {
1950                        return Ok(vec![control_error_frame(
1951                            &frame,
1952                            "unsupported_control_frame",
1953                            "module-originated channel-0 REQUEST is not supported",
1954                        )?]);
1955                    }
1956                    let request = match parse_module_control_request_from_module(&frame.body) {
1957                        Ok(request) => request,
1958                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1959                            return Ok(vec![control_error_frame(
1960                                &frame,
1961                                "unsupported_control_frame",
1962                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1963                            )?])
1964                        }
1965                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1966                            return Ok(vec![control_error_frame(
1967                                &frame,
1968                                "invalid_control_body",
1969                                format!("malformed module control body: {err}"),
1970                            )?])
1971                        }
1972                    };
1973                    let op = module_control_request_op(&request);
1974                    let corr = frame.header.corr;
1975                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1976                    let result =
1977                        self.handle_module_control_request(ctx.connection_id, frame, request);
1978                    log_slow_control_dispatch(
1979                        dispatch_started_at,
1980                        op,
1981                        ctx.connection_id,
1982                        corr,
1983                        None,
1984                    );
1985                    return result;
1986                }
1987
1988                if is_known_module_request_op(&frame.body) {
1989                    return Ok(vec![control_error_frame(
1990                        &frame,
1991                        "not_registered",
1992                        "catalog.update requires an active module registration owned by this connection",
1993                    )?]);
1994                }
1995
1996                let request = match parse_client_control_request(&frame.body) {
1997                    Ok(request) => request,
1998                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1999                        return Ok(vec![control_error_frame(
2000                            &frame,
2001                            "unknown_control_op",
2002                            format!("unknown client control op: {err}"),
2003                        )?])
2004                    }
2005                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2006                        return Ok(vec![control_error_frame(
2007                            &frame,
2008                            "invalid_control_body",
2009                            format!("malformed client control body: {err}"),
2010                        )?])
2011                    }
2012                };
2013                let op = client_control_request_op(&request);
2014                let corr = frame.header.corr;
2015                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2016                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2017                #[cfg(test)]
2018                if let Some(delay) = self.control_dispatch_delay {
2019                    tokio::time::sleep(delay).await;
2020                }
2021                let result = self
2022                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2023                    .await;
2024                if let Some(timings) = route_open_timings.as_mut() {
2025                    timings.finish();
2026                }
2027                log_slow_control_dispatch(
2028                    dispatch_started_at,
2029                    op,
2030                    ctx.connection_id,
2031                    corr,
2032                    route_open_timings.as_ref(),
2033                );
2034                result
2035            }
2036            FrameType::Push => {
2037                let Some(endpoint) = self
2038                    .forwarding
2039                    .module_endpoint_for_connection(ctx.connection_id)
2040                    .map_err(RouterError::Forwarding)?
2041                else {
2042                    return Ok(vec![control_error_frame(
2043                        &frame,
2044                        "unsupported_control_frame",
2045                        "client-originated channel-0 PUSH is not supported",
2046                    )?]);
2047                };
2048                self.handle_status_update(endpoint, frame)
2049            }
2050            FrameType::Response | FrameType::Error
2051                if self
2052                    .forwarding
2053                    .module_endpoint_for_connection(ctx.connection_id)
2054                    .map_err(RouterError::Forwarding)?
2055                    .is_some() =>
2056            {
2057                self.handle_module_relay_response(ctx.connection_id, frame)
2058            }
2059            ty => Ok(vec![control_error_frame(
2060                &frame,
2061                "unsupported_control_frame",
2062                format!("unsupported channel-0 frame {ty:?}"),
2063            )?]),
2064        }
2065    }
2066
2067    pub fn cleanup_connection(
2068        &self,
2069        connection_id: ConnectionId,
2070    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2071        self.caller_endpoints
2072            .lock()
2073            .unwrap_or_else(|p| p.into_inner())
2074            .remove(&connection_id);
2075        self.build_class_warned
2076            .lock()
2077            .unwrap_or_else(|p| p.into_inner())
2078            .remove(&connection_id);
2079        self.observed_peers
2080            .lock()
2081            .unwrap_or_else(|p| p.into_inner())
2082            .remove(&connection_id);
2083        self.operator_apps
2084            .lock()
2085            .unwrap_or_else(|p| p.into_inner())
2086            .remove(&connection_id);
2087        self.forwarding
2088            .operator_confirms()
2089            .connection_closed(connection_id);
2090        self.cleanup_connection_with_end_reason(
2091            connection_id,
2092            RegistrationEndReason::ConnectionClosed,
2093        )
2094    }
2095
2096    fn cleanup_connection_with_end_reason(
2097        &self,
2098        connection_id: ConnectionId,
2099        requested_reason: RegistrationEndReason,
2100    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2101        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2102            self.registry
2103                .get_module_by_connection(connection_id)?
2104                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2105                .and_then(|module| module.registration_end_reason().ok().flatten())
2106                .unwrap_or(requested_reason)
2107        } else {
2108            requested_reason
2109        };
2110        let crash_closed = self
2111            .registry
2112            .get_module_by_connection(connection_id)?
2113            .and_then(|registration| {
2114                self.forwarding
2115                    .module_endpoint_for_connection(connection_id)
2116                    .ok()
2117                    .flatten()
2118                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2119                    .map(|routes| (registration.manifest.module_id, routes))
2120            });
2121        let crash_closed = crash_closed.map(|(module_id, routes)| {
2122            let terminal = match self.supervisor.get(&module_id) {
2123                None => false,
2124                Some(module) => match module.will_recover_after_connection_loss() {
2125                    Ok(will_recover) => !will_recover,
2126                    Err(err) => {
2127                        warn!(
2128                            %module_id,
2129                            error = %err,
2130                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2131                        );
2132                        false
2133                    }
2134                },
2135            };
2136            // The forwarding table gates all providers at the start of daemon
2137            // shutdown, before their connections are closed. An ordinary
2138            // module disconnect still reports crash if that gate is not set.
2139            let (reason, drained) = match self
2140                .forwarding
2141                .daemon_drain_result_for_connection(connection_id)
2142            {
2143                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2144                Ok(None) => (RouteCloseReason::Crash, false),
2145                Err(err) => {
2146                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2147                    (RouteCloseReason::Crash, false)
2148                }
2149            };
2150            (module_id, routes, reason, drained, terminal)
2151        });
2152        let registrations = self.deregister_connection(connection_id, end_reason);
2153        let cleanup = if crash_closed.is_some() {
2154            self.forwarding.cleanup_connection_counted(connection_id)
2155        } else {
2156            // A module connection's teardown needs the count of abandoned
2157            // route.bind relays for its route.closed notice below. Any other
2158            // connection, such as a client's, sends no such notice and needs
2159            // only its routes released, so it uses the route-only wrapper and
2160            // reports zero.
2161            self.forwarding
2162                .cleanup_connection(connection_id)
2163                .map(|released| crate::forwarding::ConnectionCleanup {
2164                    released,
2165                    abandoned_relays: 0,
2166                })
2167        };
2168        // The route.closed push waits for forwarding teardown because only
2169        // teardown knows how many pending route.bind relays it aborted. It still
2170        // goes out before the GOODBYEs for the released routes, and its targets
2171        // were captured above, before teardown removed those routes.
2172        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2173            let abandoned = cleanup
2174                .as_ref()
2175                .map_or(0, |cleanup| cleanup.abandoned_relays);
2176            send_route_control_pushes(
2177                &self.forwarding,
2178                routes,
2179                ClientControlPush::RouteClosed {
2180                    module_id,
2181                    channels: Vec::new(),
2182                    reason,
2183                    drained,
2184                    abandoned,
2185                    excluded_subscriptions: 0,
2186                    terminal: Some(terminal),
2187                },
2188            );
2189        }
2190        if let Ok(cleanup) = cleanup {
2191            self.emit_route_goodbyes(cleanup.released);
2192        }
2193        // Signal the registration-release watch only now that BOTH registry and
2194        // forwarding teardown are done, so a supervisor waiting to spawn a
2195        // replacement never observes release while old routes still exist.
2196        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2197            crate::supervise::notify_registration_release();
2198            self.capability_evaluator.wake_deadline_loop();
2199            self.refresh_capability_requirements();
2200        }
2201        self.supervisor.remove_spawn_subscribers(connection_id);
2202        self.activity_streams
2203            .lock()
2204            .unwrap_or_else(|p| p.into_inner())
2205            .retain(|(id, _), task| {
2206                if *id == connection_id {
2207                    task.abort();
2208                    false
2209                } else {
2210                    true
2211                }
2212            });
2213        // Sync authority dies with its connection, so the owner's next
2214        // connection can take it; the owner's scopes stay as they are.
2215        self.hello_launch_nonces
2216            .lock()
2217            .unwrap_or_else(|poisoned| poisoned.into_inner())
2218            .forget(connection_id);
2219        self.scopes
2220            .write()
2221            .unwrap_or_else(|poisoned| poisoned.into_inner())
2222            .release_connection(connection_id);
2223        registrations
2224    }
2225
2226    pub(crate) fn handle_route_goodbye(
2227        &self,
2228        connection_id: ConnectionId,
2229        route_channel: u16,
2230        route_epoch: u32,
2231    ) -> Result<bool, RouterError> {
2232        debug!(
2233            connection_id = connection_id.get(),
2234            route_channel, route_epoch, "handling route GOODBYE"
2235        );
2236        let RouteRelease::Removed(released_route) = self
2237            .forwarding
2238            .release_client_route(connection_id, route_channel, route_epoch)
2239            .map_err(RouterError::Forwarding)?
2240        else {
2241            return Ok(false);
2242        };
2243        self.emit_route_goodbyes(vec![released_route]);
2244        Ok(true)
2245    }
2246
2247    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2248        for released in released_routes {
2249            let frame = match Frame::build_with_version(
2250                released.negotiated_ver,
2251                FrameType::Goodbye,
2252                control_flags(),
2253                released.channel,
2254                released.epoch,
2255                0,
2256                Vec::new(),
2257            ) {
2258                Ok(frame) => frame,
2259                Err(err) => {
2260                    warn!(
2261                        route_channel = released.channel,
2262                        error = %err,
2263                        "failed to build route GOODBYE frame"
2264                    );
2265                    continue;
2266                }
2267            };
2268            if !released.close_on_delivery_failure() {
2269                crate::forwarding::send_module_route_goodbye(
2270                    &self.counters,
2271                    &released.sink,
2272                    frame,
2273                    released.module_id.as_deref(),
2274                    "client route released",
2275                );
2276                continue;
2277            }
2278            if let Err(err) = released.sink.try_send(frame) {
2279                warn!(
2280                    target_connection_id = released.connection_id.get(),
2281                    route_channel = released.channel,
2282                    error = %err,
2283                    "route GOODBYE was not delivered to client; closing target connection"
2284                );
2285                if self
2286                    .forwarding
2287                    .escalate_client_delivery_failure(
2288                        released.connection_id,
2289                        released.channel,
2290                        released.epoch,
2291                        CloseReason::new(
2292                            "route_goodbye_delivery_failed",
2293                            format!(
2294                                "failed to enqueue route GOODBYE for channel {}: {err}",
2295                                released.channel
2296                            ),
2297                        ),
2298                        crate::forwarding::UndeliveredFrame {
2299                            module_id: released.module_id.as_deref(),
2300                            sink: &released.sink,
2301                        },
2302                    )
2303                    .unwrap_or(false)
2304                {
2305                    self.counters.increment_goodbye_relay_client_failed();
2306                }
2307            }
2308        }
2309    }
2310
2311    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2312    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2313    /// own commit failed after the module had already accepted). Without this, a
2314    /// module that accepts late keeps a binding subc has torn down, so a later
2315    /// frame on that module channel could misdeliver if the channel is reused.
2316    ///
2317    /// Never closes the shared module connection on failure: a dropped notification
2318    /// only wastes a bounded amount of warm module-side state, which the module's
2319    /// own idle reaper reclaims. Only call this once the route.bind relay was
2320    /// actually enqueued to the module — if the relay send itself failed, the
2321    /// module never created a binding and there is nothing to tear down.
2322    fn send_abandoned_route_bind_goodbye(
2323        &self,
2324        module_sink: &crate::FrameSink,
2325        negotiated_ver: u8,
2326        module_channel: u16,
2327        module_epoch: u32,
2328    ) {
2329        let frame = match Frame::build_with_version(
2330            negotiated_ver,
2331            FrameType::Goodbye,
2332            control_flags(),
2333            module_channel,
2334            module_epoch,
2335            0,
2336            Vec::new(),
2337        ) {
2338            Ok(frame) => frame,
2339            Err(err) => {
2340                warn!(
2341                    route_channel = module_channel,
2342                    error = %err,
2343                    "failed to build GOODBYE for abandoned route.bind"
2344                );
2345                return;
2346            }
2347        };
2348        crate::forwarding::send_module_route_goodbye(
2349            &self.counters,
2350            module_sink,
2351            frame,
2352            None,
2353            "abandoned route.bind",
2354        );
2355    }
2356
2357    fn handle_hello(
2358        &self,
2359        connection_id: ConnectionId,
2360        sink: Option<crate::FrameSink>,
2361        frame: Frame,
2362    ) -> Result<Vec<Frame>, RouterError> {
2363        debug!(
2364            connection_id = connection_id.get(),
2365            corr = frame.header.corr,
2366            "handling HELLO"
2367        );
2368        // A module connection has one identity for its entire lifetime. A second
2369        // registration would leave the old registry owner behind while replacing
2370        // its forwarding endpoint and launch nonce.
2371        if self
2372            .registry
2373            .get_module_by_connection(connection_id)
2374            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2375            .is_some()
2376        {
2377            return Ok(vec![control_error_frame(
2378                &frame,
2379                "invalid_hello",
2380                "connection is already registered as a module",
2381            )?]);
2382        }
2383        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2384            Ok(value) => value,
2385            Err(err) => {
2386                return Ok(vec![control_error_frame(
2387                    &frame,
2388                    "invalid_hello",
2389                    format!("malformed HELLO body: {err}"),
2390                )?])
2391            }
2392        };
2393        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2394            return Ok(vec![control_error_frame(
2395                &frame,
2396                "invalid_capability_grammar",
2397                err.to_string(),
2398            )?]);
2399        }
2400        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2401            return Ok(vec![control_error_frame(
2402                &frame,
2403                "invalid_manifest",
2404                err.to_string(),
2405            )?]);
2406        }
2407        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2408            return Ok(vec![control_error_frame(
2409                &frame,
2410                "invalid_event_declaration",
2411                err.to_string(),
2412            )?]);
2413        }
2414        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2415            return Ok(vec![control_error_frame(
2416                &frame,
2417                "invalid_manifest",
2418                err.to_string(),
2419            )?]);
2420        }
2421        if let Some(provenance) = hello_value
2422            .get("manifest")
2423            .and_then(|manifest| manifest.get("provenance"))
2424        {
2425            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2426                return Ok(vec![control_error_frame(
2427                    &frame,
2428                    "invalid_manifest",
2429                    format!("malformed manifest provenance: {err}"),
2430                )?]);
2431            }
2432        }
2433        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2434            let module_id = hello_value["manifest"]["module_id"]
2435                .as_str()
2436                .unwrap_or("<unknown>");
2437            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2438        }
2439        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2440            let module_id = hello_value["manifest"]["module_id"]
2441                .as_str()
2442                .unwrap_or("<unknown>");
2443            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2444        }
2445        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2446            let module_id = hello_value["manifest"]["module_id"]
2447                .as_str()
2448                .unwrap_or("<unknown>");
2449            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2450        }
2451        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2452            Ok(hello) => hello,
2453            Err(err) => {
2454                return Ok(vec![control_error_frame(
2455                    &frame,
2456                    "invalid_hello",
2457                    format!("malformed HELLO body: {err}"),
2458                )?])
2459            }
2460        };
2461
2462        if hello.protocol_ver != hello.manifest.protocol_ver {
2463            return Ok(vec![control_error_frame(
2464                &frame,
2465                "invalid_manifest",
2466                format!(
2467                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2468                    hello.protocol_ver, hello.manifest.protocol_ver
2469                ),
2470            )?]);
2471        }
2472
2473        if hello.manifest.module_id.trim().is_empty() {
2474            return Ok(vec![control_error_frame(
2475                &frame,
2476                "invalid_manifest",
2477                "manifest module_id must not be empty",
2478            )?]);
2479        }
2480
2481        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2482            Ok(negotiated_ver) => negotiated_ver,
2483            Err(message) => {
2484                return Ok(vec![control_error_frame(
2485                    &frame,
2486                    "version_unsupported",
2487                    message,
2488                )?])
2489            }
2490        };
2491
2492        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2493        // swap is open for this id, the only HELLO admitted as a second process
2494        // is the one carrying the candidate's launch nonce (the swap token), and
2495        // it registers into the candidate slot rather than being refused as a
2496        // duplicate. Run after the reserved gate, a reserved module's candidate
2497        // would be refused `reserved_module` for presenting a nonce that gate
2498        // does not know. See `SupervisorHandle::swap_hello_admission`.
2499        let swap_admission = self
2500            .supervisor
2501            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2502        if swap_admission == SwapHelloAdmission::Refused {
2503            warn!(
2504                module_id = %hello.manifest.module_id,
2505                connection_id = connection_id.get(),
2506                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2507            );
2508            return Ok(vec![control_error_frame(
2509                &frame,
2510                "swap_token_invalid",
2511                format!(
2512                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2513                    hello.manifest.module_id
2514                ),
2515            )?]);
2516        }
2517        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2518
2519        // A swap candidate proves its identity with the candidate nonce above.
2520        // Other registrations receive this catalog fact only when reserved-module
2521        // policy required and accepted a nonce; unreserved registrations require
2522        // no nonce check.
2523        let nonce_verified = if swap_candidate {
2524            true
2525        } else {
2526            match self.supervisor.reserved_hello_verification(
2527                &hello.manifest.module_id,
2528                hello.launch_nonce.as_deref(),
2529            ) {
2530                Ok(verified) => verified,
2531                Err(rejection) => {
2532                    let message = match rejection {
2533                        ReservedHelloRejection::Exact { module_id } => format!(
2534                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2535                        ),
2536                        ReservedHelloRejection::Prefix {
2537                            prefix,
2538                            owner_module_id,
2539                        } => format!(
2540                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2541                            hello.manifest.module_id
2542                        ),
2543                    };
2544                    return Ok(vec![control_error_frame(
2545                        &frame,
2546                        "reserved_module",
2547                        message,
2548                    )?]);
2549                }
2550            }
2551        };
2552
2553        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2554            &hello.manifest.module_id,
2555            hello.manifest.capabilities.as_ref(),
2556        );
2557        if let Some(refusal) = reserved_capability_refusals.first() {
2558            let capability = refusal.capability.clone();
2559            let bound_module = refusal.claimants[0].clone();
2560            log_duplicate_claim_events(reserved_capability_refusals);
2561            return Ok(vec![control_error_frame(
2562                &frame,
2563                "reserved_capability",
2564                format!(
2565                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2566                    capability, bound_module, hello.manifest.module_id
2567                ),
2568            )?]);
2569        }
2570
2571        // A connection that already opened client routes must not also register as
2572        // a module: cleanup would then release only one side and leak the other.
2573        if self
2574            .forwarding
2575            .connection_has_client_routes(connection_id)
2576            .map_err(RouterError::Forwarding)?
2577        {
2578            return Ok(vec![control_error_frame(
2579                &frame,
2580                "invalid_hello",
2581                "connection has open client routes and cannot also register as a module",
2582            )?]);
2583        }
2584
2585        // Kept for scope sync authority, which goes only to the connection that
2586        // presented the module's current launch nonce. Recorded before the
2587        // registration is attempted: a connection whose registration then fails
2588        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2589        self.hello_launch_nonces
2590            .lock()
2591            .unwrap_or_else(|poisoned| poisoned.into_inner())
2592            .record(connection_id, hello.launch_nonce.as_deref());
2593        let control_ops = effective_module_control_ops(hello.control_ops);
2594        // Built before anything is registered so an encoding failure leaves no
2595        // registry or forwarding state behind.
2596        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2597        if swap_candidate {
2598            return self.register_swap_candidate(
2599                connection_id,
2600                sink,
2601                &frame,
2602                hello.manifest,
2603                negotiated_ver,
2604                control_ops,
2605                hello_ack,
2606            );
2607        }
2608        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2609        // is refused, but `catalog.list` does not show the module and the
2610        // catalog generation does not move until it is published below, once
2611        // its forwarding endpoint exists. Listing it now would advertise a
2612        // module that `route.open` still refuses for having no forwarding
2613        // connection.
2614        let registration = match self
2615            .registry
2616            .register_unpublished_verified_with_control_ops(
2617                hello.manifest,
2618                negotiated_ver,
2619                connection_id,
2620                control_ops,
2621                nonce_verified,
2622            ) {
2623            Ok(registration) => registration,
2624            Err(RegistryError::DuplicateModuleId { module_id }) => {
2625                return Ok(vec![control_error_frame(
2626                    &frame,
2627                    "duplicate_module_id",
2628                    format!(
2629                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2630                    ),
2631                )?])
2632            }
2633            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2634                return Ok(vec![control_error_frame(
2635                    &frame,
2636                    "invalid_module_id",
2637                    err.to_string(),
2638                )?])
2639            }
2640            Err(err) => {
2641                return Ok(vec![control_error_frame(
2642                    &frame,
2643                    "registry_error",
2644                    err.to_string(),
2645                )?])
2646            }
2647        };
2648        #[cfg(test)]
2649        self.hold_between_registry_and_forwarding();
2650
2651        let reply = if let Some(sink) = sink {
2652            // The forwarding table's module store is also the daemon-to-module
2653            // control-RPC lane, so every HELLO gets a live endpoint even when the
2654            // manifest has no routable provider role. Non-routable modules still
2655            // cannot receive route.bind in production: `handle_route_open` checks
2656            // the registry manifest with `target_has_required_role` before the
2657            // only production call to `begin_route_bind_relay_for` below that
2658            // route.open path. The remaining direct relay callers are unit tests
2659            // and benchmark harnesses that construct forwarding state explicitly.
2660            //
2661            // The HELLO_ACK is queued by the forwarding table itself, before the
2662            // endpoint becomes visible, and is NOT returned as a reply. A module
2663            // reads HELLO_ACK first and exits on anything else; a reply is only
2664            // written after this handler returns, by which time a route.open on
2665            // another connection could already have queued a route.bind request
2666            // for this module ahead of it.
2667            let concurrency = manifest_concurrency(&registration.manifest);
2668            if let Err(err) = self.forwarding.register_module_connection_acked(
2669                connection_id,
2670                registration.manifest.module_id.clone(),
2671                negotiated_ver,
2672                concurrency,
2673                sink,
2674                hello_ack,
2675            ) {
2676                // Forwarding registration failed, so there is no forwarding
2677                // state to tear down. Remove the registry entry, which was never
2678                // published and so never listed or counted in the catalog
2679                // generation, and signal the release watch directly.
2680                if matches!(
2681                    self.deregister_connection(
2682                        connection_id,
2683                        RegistrationEndReason::RegistrationFailed,
2684                    ),
2685                    Ok(r) if !r.is_empty()
2686                ) {
2687                    crate::supervise::notify_registration_release();
2688                }
2689                return Ok(vec![control_error_frame(
2690                    &frame,
2691                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2692                        "invalid_hello"
2693                    } else {
2694                        forwarding_error_code(&err)
2695                    },
2696                    err.to_string(),
2697                )?]);
2698            }
2699            Vec::new()
2700        } else {
2701            // No sink means no forwarding endpoint, so nothing can be routed
2702            // ahead of the ack; it goes out as the reply.
2703            vec![hello_ack]
2704        };
2705
2706        // The module is routable now (or, without a sink, as routable as it
2707        // will ever be), so the catalog may list it. Publishing bumps the
2708        // catalog generation in the same registry critical section that makes
2709        // the module listed.
2710        if !self
2711            .registry
2712            .publish_registration(connection_id)
2713            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2714        {
2715            debug!(
2716                module_id = %registration.manifest.module_id,
2717                connection_id = connection_id.get(),
2718                "registration was removed before it could be published"
2719            );
2720        }
2721
2722        // Exposure over assumption: Concurrency's serde default is pinned to the
2723        // pre-field behavior (ModuleManaged), so a management surface that is
2724        // genuinely Serial and just never declared it inherits concurrent
2725        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2726        // turns "no module has been bitten yet" into the checkable claim "no
2727        // module is exposed" -- one read of the boot log instead of a fleet
2728        // audit. Detected from the raw HELLO bytes because the serde default
2729        // deliberately erases the absent/declared distinction from the type.
2730        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2731            info!(
2732                module_id = %registration.manifest.module_id,
2733                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2734            );
2735        }
2736
2737        self.apply_registration_capabilities(&registration);
2738
2739        self.supervisor
2740            .record_activity_registration(&registration.manifest.module_id);
2741
2742        info!(
2743            module_id = %registration.manifest.module_id,
2744            module_version = %registration.manifest.module_version,
2745            negotiated_ver,
2746            routable_provider = manifest_provides_routable_role(&registration.manifest),
2747            connection_id = connection_id.get(),
2748            "module registered"
2749        );
2750
2751        Ok(reply)
2752    }
2753
2754    /// Register a HELLO the swap gate admitted into the candidate slot of the
2755    /// registry and of forwarding, where it is reachable over its own
2756    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2757    /// nothing routes to it until the supervisor cuts over.
2758    ///
2759    /// Registry first (unpublished), then forwarding, then publication, the
2760    /// same order as an ordinary HELLO; a forwarding failure removes the
2761    /// registry entry again. The swap supervisor treats a published, ready
2762    /// candidate as registered and promotes it, and promotion needs the
2763    /// candidate's forwarding endpoint, so the candidate is published only
2764    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2765    /// recompute capability verdicts or run the capability deny census
2766    /// (`enforce_capability_denies`): both are about the module in the active
2767    /// slot, which `route.open` can target. A candidate sits in the candidate
2768    /// slot, so it neither provides capabilities to other modules nor can be
2769    /// routed to until promotion moves it into the active slot.
2770    #[allow(clippy::too_many_arguments)]
2771    fn register_swap_candidate(
2772        &self,
2773        connection_id: ConnectionId,
2774        sink: Option<crate::FrameSink>,
2775        frame: &Frame,
2776        manifest: ModuleManifest,
2777        negotiated_ver: u8,
2778        control_ops: Vec<String>,
2779        hello_ack: Frame,
2780    ) -> Result<Vec<Frame>, RouterError> {
2781        let module_id = manifest.module_id.clone();
2782        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2783            manifest,
2784            negotiated_ver,
2785            connection_id,
2786            control_ops,
2787        ) {
2788            Ok(registration) => registration,
2789            Err(RegistryError::DuplicateModuleId { module_id }) => {
2790                return Ok(vec![control_error_frame(
2791                    frame,
2792                    "duplicate_module_id",
2793                    format!(
2794                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2795                    ),
2796                )?])
2797            }
2798            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2799                return Ok(vec![control_error_frame(
2800                    frame,
2801                    "invalid_module_id",
2802                    err.to_string(),
2803                )?])
2804            }
2805            Err(err) => {
2806                return Ok(vec![control_error_frame(
2807                    frame,
2808                    "registry_error",
2809                    err.to_string(),
2810                )?])
2811            }
2812        };
2813        #[cfg(test)]
2814        self.hold_between_registry_and_forwarding();
2815        let reply = if let Some(sink) = sink {
2816            // Same ordering as an ordinary HELLO: the forwarding table queues
2817            // the HELLO_ACK before the candidate endpoint is inserted, because
2818            // a module exits if its first frame after HELLO is anything else.
2819            let concurrency = manifest_concurrency(&registration.manifest);
2820            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2821                connection_id,
2822                module_id.clone(),
2823                negotiated_ver,
2824                concurrency,
2825                sink,
2826                hello_ack,
2827            ) {
2828                if matches!(
2829                    self.deregister_connection(
2830                        connection_id,
2831                        RegistrationEndReason::RegistrationFailed,
2832                    ),
2833                    Ok(r) if !r.is_empty()
2834                ) {
2835                    crate::supervise::notify_registration_release();
2836                }
2837                return Ok(vec![control_error_frame(
2838                    frame,
2839                    forwarding_error_code(&err),
2840                    err.to_string(),
2841                )?]);
2842            }
2843            Vec::new()
2844        } else {
2845            vec![hello_ack]
2846        };
2847        if !self
2848            .registry
2849            .publish_registration(connection_id)
2850            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2851        {
2852            debug!(
2853                module_id = %module_id,
2854                connection_id = connection_id.get(),
2855                "swap candidate registration was removed before it could be published"
2856            );
2857        }
2858        self.supervisor.mark_swap_candidate_admitted(&module_id);
2859        info!(
2860            module_id = %module_id,
2861            module_version = %registration.manifest.module_version,
2862            negotiated_ver,
2863            ready = registration.ready,
2864            connection_id = connection_id.get(),
2865            "swap candidate registered; not routable until cutover"
2866        );
2867        Ok(reply)
2868    }
2869
2870    fn build_hello_ack(
2871        &self,
2872        frame: &Frame,
2873        negotiated_ver: u8,
2874        module_id: &str,
2875    ) -> Result<Frame, RouterError> {
2876        let ack = ModuleHelloAckBody {
2877            negotiated_ver,
2878            subc_ops: module_subc_ops(),
2879            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2880            storage: self
2881                .storage_config
2882                .as_ref()
2883                .map(|cfg| cfg.descriptor_for(module_id)),
2884            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2885        };
2886        let body = serde_json::to_vec(&ack).map_err(|err| {
2887            RouterError::backend(
2888                0,
2889                frame.header.corr,
2890                format!("failed to encode HELLO_ACK: {err}"),
2891            )
2892        })?;
2893
2894        Frame::build_with_version(
2895            negotiated_ver,
2896            FrameType::HelloAck,
2897            control_flags(),
2898            0,
2899            0,
2900            frame.header.corr,
2901            body,
2902        )
2903        .map_err(RouterError::FrameBuild)
2904    }
2905
2906    async fn handle_client_control_request(
2907        &self,
2908        ctx: &RouteCtx,
2909        frame: Frame,
2910        request: ClientControlRequest,
2911        route_open_timings: &mut Option<RouteOpenTimings>,
2912    ) -> Result<Vec<Frame>, RouterError> {
2913        if let Some(operation) = privileged_operation(&request) {
2914            let caller = self.resolve_caller(ctx.connection_id).await;
2915            let op = client_control_request_op(&request);
2916            let text = prompt_text(operation, &caller);
2917            match self.privileged_operator_prompt {
2918                PrivilegedOperatorPrompt::Log => {
2919                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2920                        target = ?operation, caller = %caller.description(), summary = %text,
2921                        "privileged request would prompt the operator");
2922                }
2923                PrivilegedOperatorPrompt::Enforce => {
2924                    let key =
2925                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2926                    let outcome = self
2927                        .forwarding
2928                        .operator_confirms()
2929                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2930                        .await;
2931                    if outcome.code != "confirmed" {
2932                        return Ok(vec![outcome.frame(&frame)]);
2933                    }
2934                }
2935            }
2936        }
2937        match request {
2938            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2939            ClientControlRequest::CatalogList { module_id } => {
2940                self.handle_catalog_list(frame, module_id)
2941            }
2942            ClientControlRequest::RouteOpen {
2943                target,
2944                identity,
2945                consumer_identity,
2946                consumer_capabilities,
2947                role_versions,
2948                admission_facts,
2949                scope,
2950            } => {
2951                self.handle_route_open(
2952                    ctx,
2953                    frame,
2954                    RouteOpenRequest {
2955                        target,
2956                        identity,
2957                        consumer_identity,
2958                        consumer_capabilities,
2959                        role_versions,
2960                        admission_facts,
2961                        scope,
2962                    },
2963                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2964                )
2965                .await
2966            }
2967            ClientControlRequest::RoutePoll {
2968                route_channel,
2969                route_epoch,
2970                kind,
2971            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2972            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2973            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2974                self.handle_supervisor_spawn_snapshot(frame)
2975            }
2976            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2977                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2978            }
2979            ClientControlRequest::SupervisorRestart {
2980                module_id,
2981                drain_timeout_ms,
2982            } => {
2983                self.log_supervisor_request_received(
2984                    ctx,
2985                    frame.header.corr,
2986                    ops::SUPERVISOR_RESTART,
2987                    Some(&module_id),
2988                    None,
2989                )?;
2990                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2991                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2992                    .await
2993            }
2994            ClientControlRequest::SupervisorSwap {
2995                module_id,
2996                ready_timeout_ms,
2997            } => {
2998                self.log_supervisor_request_received(
2999                    ctx,
3000                    frame.header.corr,
3001                    ops::SUPERVISOR_SWAP,
3002                    Some(&module_id),
3003                    None,
3004                )?;
3005                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3006                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3007                    .await
3008            }
3009            ClientControlRequest::SupervisorReload { module_id } => {
3010                self.log_supervisor_request_received(
3011                    ctx,
3012                    frame.header.corr,
3013                    ops::SUPERVISOR_RELOAD,
3014                    Some(&module_id),
3015                    None,
3016                )?;
3017                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3018                self.handle_supervisor_reload(frame, module_id, requester)
3019                    .await
3020            }
3021            ClientControlRequest::SupervisorRescan { preview } => {
3022                if !preview {
3023                    self.log_supervisor_request_received(
3024                        ctx,
3025                        frame.header.corr,
3026                        ops::SUPERVISOR_RESCAN,
3027                        None,
3028                        None,
3029                    )?;
3030                }
3031                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3032                self.handle_supervisor_rescan(
3033                    frame,
3034                    if preview {
3035                        RescanMode::Preview
3036                    } else {
3037                        RescanMode::Execute
3038                    },
3039                    requester,
3040                )
3041                .await
3042            }
3043            ClientControlRequest::SupervisorRescanChecked {
3044                preview,
3045                candidate,
3046                expect,
3047            } => {
3048                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
3049                    .await
3050            }
3051            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3052                self.log_supervisor_request_received(
3053                    ctx,
3054                    frame.header.corr,
3055                    ops::SUPERVISOR_RELEASE_RESERVED,
3056                    Some(&module_id),
3057                    None,
3058                )?;
3059                self.handle_supervisor_release_reserved(frame, module_id)
3060                    .await
3061            }
3062            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3063                self.log_supervisor_request_received(
3064                    ctx,
3065                    frame.header.corr,
3066                    ops::SUPERVISOR_SET_ENABLED,
3067                    Some(&module_id),
3068                    Some(enabled),
3069                )?;
3070                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3071                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3072                    .await
3073            }
3074            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3075                self.handle_supervisor_health_probe(frame, module_id).await
3076            }
3077            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3078            ClientControlRequest::SupervisorRoutes { module_id } => {
3079                self.handle_supervisor_routes(frame, module_id)
3080            }
3081            ClientControlRequest::SupervisorProvenance { module_id } => {
3082                self.handle_supervisor_provenance(frame, module_id).await
3083            }
3084            ClientControlRequest::SupervisorStderrTail {
3085                module_id,
3086                max_lines,
3087                max_bytes,
3088            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3089            ClientControlRequest::SupervisorTerminals { module_id } => {
3090                self.handle_supervisor_terminals(frame, module_id).await
3091            }
3092        }
3093    }
3094
3095    fn log_supervisor_request_received(
3096        &self,
3097        ctx: &RouteCtx,
3098        corr: u64,
3099        op: &'static str,
3100        module_id: Option<&str>,
3101        enabled: Option<bool>,
3102    ) -> Result<(), RouterError> {
3103        let caller = self.supervisor_requester(ctx, corr)?;
3104        let caller = principal_label(&caller);
3105        // `Option` fields are recorded only when present, so a request with no
3106        // module id (rescan) or no enabled flag simply omits that field.
3107        info!(
3108            target: "control",
3109            op,
3110            module_id,
3111            enabled,
3112            connection_id = ctx.connection_id.get(),
3113            caller = %caller,
3114            "supervisor request received"
3115        );
3116        Ok(())
3117    }
3118
3119    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3120        Ok(self
3121            .registry
3122            .get_module_by_connection(ctx.connection_id)
3123            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3124            .map(|registration| Principal::Reserved {
3125                module_id: registration.manifest.module_id,
3126            })
3127            .unwrap_or(Principal::Direct))
3128    }
3129
3130    fn handle_module_control_request(
3131        &self,
3132        connection_id: ConnectionId,
3133        frame: Frame,
3134        request: ModuleControlRequestFromModule,
3135    ) -> Result<Vec<Frame>, RouterError> {
3136        match request {
3137            ModuleControlRequestFromModule::CatalogUpdate {
3138                provides,
3139                capabilities,
3140                ready,
3141            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3142            ModuleControlRequestFromModule::LiveRoots {} => {
3143                let registered = self
3144                    .registry
3145                    .get_module_by_connection(connection_id)
3146                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3147                let Some(registration) = registered else {
3148                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3149                };
3150                let response = self
3151                    .forwarding
3152                    .live_roots(&registration.manifest.module_id)
3153                    .map_err(RouterError::Forwarding)?;
3154                Ok(vec![control_response_body_frame(
3155                    &frame,
3156                    &response,
3157                    "ModuleControlResponseToModule::LiveRoots",
3158                )?])
3159            }
3160            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3161                self.handle_scope_sync(connection_id, frame, generation, scopes)
3162            }
3163            ModuleControlRequestFromModule::ScopeApply {
3164                generation,
3165                upsert,
3166                end,
3167            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3168            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3169                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3170            }
3171        }
3172    }
3173
3174    fn handle_scope_served_by(
3175        &self,
3176        connection_id: ConnectionId,
3177        frame: Frame,
3178    ) -> Result<Vec<Frame>, RouterError> {
3179        let registered = self
3180            .registry
3181            .get_module_by_connection(connection_id)
3182            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3183        let Some(registered) = registered else {
3184            return Ok(vec![control_error_frame(
3185                &frame,
3186                "not_registered",
3187                "scope.served_by requires an active module registration owned by this connection",
3188            )?]);
3189        };
3190        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3191        {
3192            Ok(request) => request,
3193            Err(err) => {
3194                return Ok(vec![control_error_frame(
3195                    &frame,
3196                    "invalid_control_body",
3197                    format!("malformed scope.served_by body: {err}"),
3198                )?])
3199            }
3200        };
3201        let answer = self
3202            .scopes
3203            .read()
3204            .unwrap_or_else(|poisoned| poisoned.into_inner())
3205            .served_by(
3206                &registered.manifest.module_id,
3207                &request.scope_ref,
3208                request.scope_epoch,
3209                &request.module_id,
3210            );
3211        let reply = subc_control::ScopeServedByReply::new(
3212            answer,
3213            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3214        );
3215        Ok(vec![control_response_body_frame(
3216            &frame,
3217            &reply,
3218            "ScopeServedByReply",
3219        )?])
3220    }
3221
3222    fn handle_operator_confirm(
3223        &self,
3224        ctx: &RouteCtx,
3225        frame: Frame,
3226    ) -> Result<Vec<Frame>, RouterError> {
3227        use crate::operator_confirm::{audit, Outcome};
3228        let registration = self
3229            .registry
3230            .get_module_by_connection(ctx.connection_id)
3231            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3232        let Some(registration) = registration else {
3233            let outcome = Outcome::refusal("not_registered");
3234            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3235            return Ok(vec![outcome.frame(&frame)]);
3236        };
3237        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3238            Ok(request) => request,
3239            Err(_) => {
3240                let outcome = Outcome::refusal("invalid_control_body");
3241                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3242                return Ok(vec![outcome.frame(&frame)]);
3243            }
3244        };
3245        let module_id = registration.manifest.module_id;
3246        // Read the launch nonce (under its own lock) before taking the forwarding
3247        // table's lock below: holding forwarding while waiting on another daemon
3248        // lock risks a lock-order deadlock with paths that take them the other way.
3249        let nonce = self
3250            .hello_launch_nonces
3251            .lock()
3252            .unwrap_or_else(|p| p.into_inner())
3253            .nonce(ctx.connection_id)
3254            .map(str::to_owned);
3255        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3256            self.supervisor
3257                .spawned_consumer_authorized(&module_id, nonce)
3258        });
3259        let confirms = self.forwarding.operator_confirms();
3260        self.forwarding
3261            .with_operator_route(
3262                ctx.connection_id,
3263                request.route_channel,
3264                request.route_epoch,
3265                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3266            )
3267            .map_err(RouterError::Forwarding)
3268    }
3269
3270    /// Resolve folders before taking the scope-table lock: a slow filesystem
3271    /// must not hold up admissions for unrelated scoped routes.
3272    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3273        let mut scopes = ScopeRecords::from(scopes);
3274        for (index, record) in scopes.records.iter_mut().enumerate() {
3275            if let Some(refusal) = Self::canonicalize_workdir(record) {
3276                scopes.refusals.insert(index, refusal);
3277            }
3278        }
3279        scopes
3280    }
3281
3282    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3283        let workdir = record.workdir.as_mut()?;
3284        let path = std::path::Path::new(workdir);
3285        if !path.is_absolute() {
3286            return Some((
3287                error_codes::INVALID_WORKDIR,
3288                "workdir must be absolute".to_string(),
3289            ));
3290        }
3291        match ProjectRootId::from_path_allowing_missing(path) {
3292            Ok(root) => match root.as_path().to_str() {
3293                Some(canonical) => {
3294                    *workdir = canonical.to_string();
3295                    None
3296                }
3297                None => Some((
3298                    error_codes::INVALID_WORKDIR,
3299                    "canonical workdir is not UTF-8".to_string(),
3300                )),
3301            },
3302            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3303        }
3304    }
3305
3306    /// `scope.sync`: the owner is the module registered on this connection.
3307    /// A connection with no registration (every client connection, `direct`
3308    /// included) is refused `not_registered` before the table is consulted.
3309    fn handle_scope_sync(
3310        &self,
3311        connection_id: ConnectionId,
3312        frame: Frame,
3313        generation: u64,
3314        scopes: Vec<ScopeRecord>,
3315    ) -> Result<Vec<Frame>, RouterError> {
3316        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3317    }
3318
3319    fn handle_scope_change(
3320        &self,
3321        connection_id: ConnectionId,
3322        frame: Frame,
3323        generation: u64,
3324        scopes: Vec<ScopeRecord>,
3325        end: Option<Vec<ScopeEnd>>,
3326    ) -> Result<Vec<Frame>, RouterError> {
3327        let is_apply = end.is_some();
3328        let Some(registration) = self
3329            .registry
3330            .get_module_by_connection(connection_id)
3331            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3332        else {
3333            return Ok(vec![control_error_frame(
3334                &frame,
3335                "not_registered",
3336                "scope.sync and scope.apply require an active module registration owned by this connection",
3337            )?]);
3338        };
3339        let owner = registration.manifest.module_id;
3340        let scopes = Self::prepare_scope_records(scopes);
3341        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3342        let is_current_launch = |connection: ConnectionId| {
3343            self.hello_launch_nonces
3344                .lock()
3345                .unwrap_or_else(|poisoned| poisoned.into_inner())
3346                .presented(connection, current_nonce.as_deref())
3347        };
3348        // Lock order is the scope table, then the forwarding table: the new
3349        // tags are published, and the routes the change closes are selected,
3350        // while the scope table is still write-locked, so no admission can read
3351        // a record whose tag is not yet published.
3352        let mut table = self
3353            .scopes
3354            .write()
3355            .unwrap_or_else(|poisoned| poisoned.into_inner());
3356        let outcome = match end {
3357            Some(end) => table.apply(
3358                &owner,
3359                connection_id,
3360                is_current_launch,
3361                generation,
3362                scopes,
3363                end,
3364            ),
3365            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3366        };
3367        let drained = match &outcome {
3368            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3369            Err(_) => Vec::new(),
3370        };
3371        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3372        drop(table);
3373        if has_deadlines {
3374            self.scope_deadline_added.notify_one();
3375        }
3376        match outcome {
3377            Ok(applied) => {
3378                let counts = ScopeOutcomeCounts::of(&applied.results);
3379                info!(
3380                    owner = %owner,
3381                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3382                    generation,
3383                    records = applied.results.len(),
3384                    created = counts.created,
3385                    replaced = counts.replaced,
3386                    updated = counts.updated,
3387                    unchanged = counts.unchanged,
3388                    refused = counts.refused,
3389                    ended = applied.ended.len(),
3390                    tag_changes = applied.tag_changes.len(),
3391                    routes_closed = drained.len(),
3392                    "scope change accepted"
3393                );
3394                // An accepted scope change can still refuse individual records, and the
3395                // owner is the only party that sees the reply. Name them here so
3396                // an operator can tell a refused session from a missing one
3397                // without the owner's logs. Capped so a call that refuses
3398                // thousands cannot flood the log; the count above is complete.
3399                for refused in applied
3400                    .results
3401                    .iter()
3402                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3403                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3404                {
3405                    warn!(
3406                        owner = %owner,
3407                        generation,
3408                        scope_ref = %refused.scope_ref,
3409                        scope_epoch = refused.scope_epoch,
3410                        code = refused.code.as_deref().unwrap_or(""),
3411                        "scope record refused"
3412                    );
3413                }
3414                self.close_scope_drained_routes(drained);
3415                let response = if is_apply {
3416                    ModuleControlResponseToModule::ScopeApply {
3417                        generation,
3418                        results: applied.results,
3419                        end_results: applied.end_results,
3420                        ended: applied.ended,
3421                    }
3422                } else {
3423                    ModuleControlResponseToModule::ScopeSync {
3424                        generation,
3425                        results: applied.results,
3426                        ended: applied.ended,
3427                    }
3428                };
3429                Ok(vec![control_response_body_frame(
3430                    &frame,
3431                    &response,
3432                    "scope change response",
3433                )?])
3434            }
3435            Err(refusal) => {
3436                info!(
3437                    owner = %owner,
3438                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3439                    generation,
3440                    code = refusal.code,
3441                    "scope change refused"
3442                );
3443                Ok(vec![control_error_frame(
3444                    &frame,
3445                    refusal.code,
3446                    refusal.message,
3447                )?])
3448            }
3449        }
3450    }
3451
3452    /// Tell both ends of each route a scope change closed. The module gets a
3453    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3454    /// the client's route handle. The client also gets `route.closed` with the
3455    /// scope reason, one push per module and reason, so it can tell a revoked
3456    /// route from an ordinary close and not reopen it.
3457    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3458        if drained.is_empty() {
3459            return;
3460        }
3461        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3462            BTreeMap::new();
3463        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3464        for route in drained {
3465            warn!(
3466                module_id = %route.module_id,
3467                reason = ?route.reason,
3468                client_connection_id = route.client.connection_id.get(),
3469                route_channel = route.client.channel,
3470                "closing route because its scope changed"
3471            );
3472            pushes
3473                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3474                .or_insert_with(|| (route.reason, Vec::new()))
3475                .1
3476                .push(EndpointRoute {
3477                    goodbye_target: route.client.clone(),
3478                    principal: Principal::Unverified,
3479                    bound_at: Instant::now(),
3480                    draining: false,
3481                    drain_reason: None,
3482                });
3483            goodbyes.push(route.module);
3484            goodbyes.push(route.client);
3485        }
3486        for ((module_id, _), (reason, routes)) in pushes {
3487            send_route_control_pushes(
3488                &self.forwarding,
3489                routes,
3490                ClientControlPush::RouteClosed {
3491                    module_id,
3492                    channels: Vec::new(),
3493                    reason,
3494                    drained: false,
3495                    abandoned: 0,
3496                    excluded_subscriptions: 0,
3497                    terminal: Some(false),
3498                },
3499            );
3500        }
3501        self.emit_route_goodbyes(goodbyes);
3502    }
3503
3504    /// `scope.describe`: any registered module may read any scope, because a
3505    /// provider must read the scope a route it serves is stamped with.
3506    fn handle_scope_describe(
3507        &self,
3508        connection_id: ConnectionId,
3509        frame: Frame,
3510        owner: Principal,
3511        scope_ref: String,
3512    ) -> Result<Vec<Frame>, RouterError> {
3513        let registered = self
3514            .registry
3515            .get_module_by_connection(connection_id)
3516            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3517        let Some(registered) = registered else {
3518            return Ok(vec![control_error_frame(
3519                &frame,
3520                "not_registered",
3521                "scope.describe requires an active module registration owned by this connection",
3522            )?]);
3523        };
3524        let mut description = self
3525            .scopes
3526            .read()
3527            .unwrap_or_else(|poisoned| poisoned.into_inner())
3528            .describe(&owner, &scope_ref);
3529        // Describe readers also decode strict ScopeAttributes. Attribution is
3530        // safe to omit unless the requesting module declares it understands it.
3531        if !registered
3532            .manifest
3533            .capabilities
3534            .as_ref()
3535            .is_some_and(|caps| {
3536                caps.provides
3537                    .iter()
3538                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3539            })
3540        {
3541            if let Some(stamp) = description.stamp.as_mut() {
3542                stamp.attributes.on_behalf_of_agent = None;
3543            }
3544        }
3545        if !registered
3546            .manifest
3547            .capabilities
3548            .as_ref()
3549            .is_some_and(|caps| {
3550                caps.provides
3551                    .iter()
3552                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3553            })
3554        {
3555            if let Some(stamp) = description.stamp.as_mut() {
3556                stamp.attributes.flow_version = None;
3557                stamp.attributes.flow_code_hash = None;
3558                stamp.attributes.flow_install_id = None;
3559            }
3560        }
3561        let owner_configured = match &owner {
3562            // Ask whether the owner is configured (`is_configured`), not
3563            // whether it is on the roster (`get(..).is_some()`): a supervised
3564            // module's process can register and describe a scope before the
3565            // supervisor has put it on the roster.
3566            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3567            _ => false,
3568        };
3569        let response = ModuleControlResponseToModule::ScopeDescribe {
3570            status: description.status,
3571            scope_epoch: description.scope_epoch,
3572            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3573            owner_synced: description.owner_synced,
3574            owner_configured,
3575            scope: description.stamp,
3576        };
3577        Ok(vec![control_response_body_frame(
3578            &frame,
3579            &response,
3580            "ModuleControlResponseToModule::ScopeDescribe",
3581        )?])
3582    }
3583
3584    fn handle_catalog_update(
3585        &self,
3586        connection_id: ConnectionId,
3587        frame: Frame,
3588        provides: Vec<ProviderRole>,
3589        capabilities: Option<CapabilityDeclarations>,
3590        ready: Option<bool>,
3591    ) -> Result<Vec<Frame>, RouterError> {
3592        self.refresh_capability_requirements();
3593        let Some(registration) = self
3594            .registry
3595            .get_module_by_connection(connection_id)
3596            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3597        else {
3598            return Ok(vec![control_error_frame(
3599                &frame,
3600                "not_registered",
3601                "catalog.update requires an active module registration owned by this connection",
3602            )?]);
3603        };
3604
3605        if let Some(message) =
3606            catalog_update_frozen_field_message(&registration.manifest, &provides)
3607        {
3608            return Ok(vec![control_error_frame(
3609                &frame,
3610                "catalog_update_frozen_field",
3611                message,
3612            )?]);
3613        }
3614
3615        let mut candidate = registration.manifest.clone();
3616        candidate.provides = provides.clone();
3617        candidate.capabilities = capabilities
3618            .clone()
3619            .or_else(|| registration.manifest.capabilities.clone());
3620        if let Some(rich_ui) = &candidate.rich_ui {
3621            if let Err(err) = rich_ui.validate() {
3622                return Ok(vec![control_error_frame(
3623                    &frame,
3624                    "invalid_manifest",
3625                    err.to_string(),
3626                )?]);
3627            }
3628        }
3629        if let Err(err) = candidate.validate_capability_grammar() {
3630            return Ok(vec![control_error_frame(
3631                &frame,
3632                "invalid_capability_grammar",
3633                err.to_string(),
3634            )?]);
3635        }
3636
3637        // Updates must honor the same reserved owner as initial registration;
3638        // otherwise an empty HELLO could acquire the claim after admission.
3639        let mut conflicts = self
3640            .capability_evaluator
3641            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3642        if let Some(conflict) = conflicts.first() {
3643            let message = format!(
3644                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3645                conflict.capability, conflict.claimants[0], candidate.module_id
3646            );
3647            for conflict in &mut conflicts {
3648                conflict.source = DuplicateClaimSource::CatalogUpdate;
3649            }
3650            log_duplicate_claim_events(conflicts);
3651            return Ok(vec![control_error_frame(
3652                &frame,
3653                "reserved_capability",
3654                message,
3655            )?]);
3656        }
3657
3658        let updated = self
3659            .registry
3660            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3661            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3662        if updated.is_none() {
3663            return Ok(vec![control_error_frame(
3664                &frame,
3665                "not_registered",
3666                "catalog.update requires an active module registration owned by this connection",
3667            )?]);
3668        }
3669        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3670            log_duplicate_claim_events(
3671                self.capability_evaluator
3672                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3673            );
3674        }
3675        if capability_census_trigger(
3676            registration.manifest.capabilities.as_ref(),
3677            updated
3678                .as_ref()
3679                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3680        ) {
3681            self.enforce_capability_denies();
3682        }
3683        self.refresh_capability_requirements();
3684
3685        let response = ModuleControlResponseToModule::CatalogUpdate {};
3686        control_response_body_frame(
3687            &frame,
3688            &response,
3689            "ModuleControlResponseToModule::CatalogUpdate",
3690        )
3691        .map(|frame| vec![frame])
3692    }
3693
3694    pub(crate) fn with_activity_ledger_config(
3695        mut self,
3696        run_dir: &Path,
3697        incarnation: String,
3698        config: crate::activity::ActivityConfig,
3699    ) -> Self {
3700        self.activity = Some(crate::activity::ActivityLedger::open_config(
3701            run_dir.join("activity.jsonl"),
3702            incarnation,
3703            config,
3704        ));
3705        self.supervisor.set_activity(self.activity.clone());
3706        self
3707    }
3708
3709    pub(crate) async fn configure_activity(
3710        &self,
3711        config: crate::activity::ActivityConfig,
3712    ) -> Result<(), &'static str> {
3713        match &self.activity {
3714            Some(ledger) => ledger.configure(config).await,
3715            None => Ok(()),
3716        }
3717    }
3718
3719    fn cancel_activity_subscription(
3720        &self,
3721        ctx: &RouteCtx,
3722        frame: &Frame,
3723    ) -> Result<bool, RouterError> {
3724        let task = self
3725            .activity_streams
3726            .lock()
3727            .unwrap_or_else(|p| p.into_inner())
3728            .remove(&(ctx.connection_id, frame.header.corr));
3729        let Some(task) = task else {
3730            return Ok(false);
3731        };
3732        task.abort();
3733        let end = Frame::build_with_version(
3734            response_version(frame),
3735            FrameType::StreamEnd,
3736            control_flags(),
3737            0,
3738            0,
3739            frame.header.corr,
3740            Vec::new(),
3741        )
3742        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3743        let sink = ctx.egress.clone();
3744        tokio::spawn(async move {
3745            let _ = sink.send(end).await;
3746        });
3747        Ok(true)
3748    }
3749
3750    async fn handle_activity_read(
3751        &self,
3752        ctx: &RouteCtx,
3753        frame: Frame,
3754    ) -> Result<Vec<Frame>, RouterError> {
3755        use subc_control::{
3756            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3757            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3758        };
3759        #[derive(Deserialize)]
3760        struct Identity {
3761            #[serde(default)]
3762            consumer_identity: Option<ConsumerIdentity>,
3763        }
3764        // A registered module connection already identifies its caller. Ignore
3765        // consumer_identity there so a module cannot borrow a mirror's identity.
3766        // Only separate consumer connections use the Direct/nonce trust boundary.
3767        let principal = match self
3768            .forwarding
3769            .module_id_for_connection(ctx.connection_id)
3770            .map_err(RouterError::Forwarding)?
3771        {
3772            Some(module_id) => Ok(Principal::Reserved { module_id }),
3773            None => {
3774                let identity: Identity = match serde_json::from_slice(&frame.body) {
3775                    Ok(identity) => identity,
3776                    Err(_) => {
3777                        return Ok(vec![control_error_frame(
3778                            &frame,
3779                            ACTIVITY_MALFORMED,
3780                            "malformed activity read",
3781                        )?])
3782                    }
3783                };
3784                self.route_open_principal(&frame, identity.consumer_identity)?
3785            }
3786        };
3787        let admitted = matches!(principal, Ok(Principal::Direct))
3788            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3789        if !admitted {
3790            return Ok(vec![control_error_frame(
3791                &frame,
3792                ACTIVITY_READ_FORBIDDEN,
3793                "activity reads require an operator or trusted mirror principal",
3794            )?]);
3795        }
3796        let Some(ledger) = &self.activity else {
3797            return Ok(vec![control_error_frame(
3798                &frame,
3799                ACTIVITY_UNAVAILABLE,
3800                "activity ledger unavailable",
3801            )?]);
3802        };
3803        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3804            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3805        {
3806            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3807                Ok(request) => ledger.query(request).await,
3808                Err(_) => Err(ACTIVITY_MALFORMED),
3809            };
3810            return match result {
3811                Ok(reply) => Ok(vec![control_response_body_frame(
3812                    &frame,
3813                    &reply,
3814                    ops::ACTIVITY_QUERY,
3815                )?]),
3816                Err(code) => Ok(vec![control_error_frame(
3817                    &frame,
3818                    code,
3819                    "activity query refused",
3820                )?]),
3821            };
3822        }
3823        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3824            Ok(request) => request,
3825            Err(_) => {
3826                return Ok(vec![control_error_frame(
3827                    &frame,
3828                    ACTIVITY_MALFORMED,
3829                    "malformed activity subscription",
3830                )?])
3831            }
3832        };
3833        let mut subscription = match ledger.subscribe(request.since).await {
3834            Ok(subscription) => subscription,
3835            Err(code) => {
3836                return Ok(vec![control_error_frame(
3837                    &frame,
3838                    code,
3839                    "activity subscription refused",
3840                )?])
3841            }
3842        };
3843        let key = (ctx.connection_id, frame.header.corr);
3844        let sink = ctx.egress.clone();
3845        let mut streams = self
3846            .activity_streams
3847            .lock()
3848            .unwrap_or_else(|p| p.into_inner());
3849        streams.retain(|_, task| !task.is_finished());
3850        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3851            return Ok(vec![control_error_frame(
3852                &frame,
3853                "duplicate_subscription",
3854                "activity subscription correlation id already in use",
3855            )?]);
3856        }
3857        let task = tokio::spawn(async move {
3858            loop {
3859                let entry = if let Some(entry) = subscription.replay.pop_front() {
3860                    Some(entry)
3861                } else {
3862                    subscription.receiver.recv().await
3863                };
3864                let Some(entry) = entry else {
3865                    break;
3866                };
3867                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3868                    return;
3869                };
3870                let Ok(next) = Frame::build_with_version(
3871                    response_version(&frame),
3872                    FrameType::StreamData,
3873                    control_flags(),
3874                    0,
3875                    0,
3876                    frame.header.corr,
3877                    body,
3878                ) else {
3879                    return;
3880                };
3881                if sink.send(next).await.is_err() {
3882                    return;
3883                }
3884            }
3885            // Dropping a full subscriber's sender drains its accepted entries
3886            // before the terminal error, preserving their order on the wire.
3887            if let Some(code) = subscription.terminal_code() {
3888                if let Ok(end) =
3889                    control_error_frame(&frame, code, "activity subscriber fell behind")
3890                {
3891                    let _ = sink.send(end).await;
3892                }
3893            }
3894        });
3895        streams.insert(key, task.abort_handle());
3896        Ok(Vec::new())
3897    }
3898
3899    async fn handle_activity_record(
3900        &self,
3901        ctx: &RouteCtx,
3902        frame: Frame,
3903    ) -> Result<Vec<Frame>, RouterError> {
3904        use subc_control::{
3905            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3906            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3907        };
3908        let result = async {
3909            let module = self
3910                .forwarding
3911                .module_id_for_connection(ctx.connection_id)
3912                .ok()
3913                .flatten()
3914                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3915            let request: ActivityRecordRequest =
3916                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3917            crate::activity::validate(&module, &request)?;
3918            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3919            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3920            let at_ms = std::time::SystemTime::now()
3921                .duration_since(std::time::UNIX_EPOCH)
3922                .unwrap_or_default()
3923                .as_millis() as u64;
3924            ledger.record(module, request, actor, at_ms).await
3925        }
3926        .await;
3927        match result {
3928            Ok(entry) => Ok(vec![control_response_body_frame(
3929                &frame,
3930                &ActivityRecordResponse { seq: entry.seq },
3931                "activity.record",
3932            )?]),
3933            Err(code) => Ok(vec![control_error_frame(
3934                &frame,
3935                code,
3936                "activity recording refused",
3937            )?]),
3938        }
3939    }
3940
3941    fn activity_actor(
3942        &self,
3943        connection: ConnectionId,
3944        module: &str,
3945        on: Option<&subc_control::ActivityOn>,
3946    ) -> Result<subc_control::ActivityActor, &'static str> {
3947        use crate::forwarding::{DataRoute, DataRouteState};
3948        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3949        let principal = |id: &str| Principal::Reserved {
3950            module_id: id.to_owned(),
3951        };
3952        let mut actor = ActivityActor {
3953            principal: principal(module),
3954            scope_owner: None,
3955            agent_id: None,
3956            run_id: None,
3957            flow_id: None,
3958            project_root: None,
3959            workdir: None,
3960            project_id: None,
3961        };
3962        let scope = match on {
3963            None => return Ok(actor),
3964            Some(ActivityOn::Scope(scope)) => {
3965                let selector = subc_protocol::scope::ScopeSelector::new(
3966                    principal(module),
3967                    scope.scope_ref.clone(),
3968                    Some(scope.epoch),
3969                );
3970                self.scopes
3971                    .read()
3972                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3973                    .attribute(&principal(module), module, &selector, true)
3974                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3975            }
3976            Some(ActivityOn::Route(route)) => {
3977                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3978                    .forwarding
3979                    .lookup_data_route(connection, route.channel, route.epoch)
3980                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3981                else {
3982                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3983                };
3984                if binding.module_id != module {
3985                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3986                }
3987                actor.principal = binding.principal.clone();
3988                actor.project_root = binding
3989                    .project_root
3990                    .as_ref()
3991                    .map(|root| root.as_path().to_path_buf());
3992                let Some(bound) = &binding.scope else {
3993                    return Ok(actor);
3994                };
3995                let owner = principal(&bound.owner);
3996                let selector = subc_protocol::scope::ScopeSelector::new(
3997                    owner.clone(),
3998                    bound.scope_ref.clone(),
3999                    Some(bound.tag.scope_epoch),
4000                );
4001                let admitted = self
4002                    .scopes
4003                    .read()
4004                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4005                    .attribute(&owner, module, &selector, true)
4006                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4007                if admitted.tag != bound.tag {
4008                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4009                }
4010                admitted
4011            }
4012        };
4013        actor.scope_owner = Some(scope.stamp.owner);
4014        actor.agent_id = scope.stamp.attributes.agent_id;
4015        actor.run_id = scope.stamp.attributes.run_id;
4016        actor.flow_id = scope.stamp.attributes.flow_id;
4017        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4018        actor.project_id = scope.stamp.project_id;
4019        Ok(actor)
4020    }
4021
4022    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4023        self.refresh_capability_requirements();
4024        // A bare connection count is ambiguous between many clients holding a
4025        // route each and one client accumulating hundreds, so publish the
4026        // concentration alongside it. Route state is best-effort here: a
4027        // diagnostic endpoint must still answer if the forwarding lock is
4028        // contended.
4029        let mut counters = self.counters.snapshot();
4030        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4031            obj.insert(
4032                "activity_unparseable_lines".into(),
4033                ledger.read_errors().into(),
4034            );
4035            obj.insert(
4036                "activity_lifecycle_append_failures".into(),
4037                ledger.lifecycle_failures().into(),
4038            );
4039            obj.insert(
4040                "activity_started_digest_skips".into(),
4041                ledger.lifecycle_digest_skips().into(),
4042            );
4043        }
4044        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4045            self.forwarding.client_route_concentration(),
4046            counters.as_object_mut(),
4047        ) {
4048            obj.insert(
4049                "client_connections_with_routes".into(),
4050                connections_with_routes.into(),
4051            );
4052            obj.insert("max_routes_on_one_connection".into(), max.into());
4053        }
4054        // A module that is being fast-refused and a module that is fine look
4055        // identical from a client that retries and succeeds, so name the open
4056        // breakers here. This rides the existing free-form counters object
4057        // rather than a new wire field, so no sibling that deserializes
4058        // `ServerDescribe` has to be rebuilt to keep reading it.
4059        if let (Some(open_breakers), Some(obj)) = (
4060            self.route_bind_breakers.open_snapshot(),
4061            counters.as_object_mut(),
4062        ) {
4063            obj.insert("route_bind_breakers_open".into(), open_breakers);
4064        }
4065        // A daemon whose local endpoint could not be bound keeps serving TCP,
4066        // and the connection file names no endpoint. Name the reason here, in
4067        // the free-form counters object, so an operator reading daemon facts
4068        // sees it without searching the log, and no wire field changes.
4069        if let (Some(reason), Some(obj)) = (
4070            self.local_endpoint_unavailable.clone(),
4071            counters.as_object_mut(),
4072        ) {
4073            obj.insert("local_endpoint_unavailable".into(), reason.into());
4074        }
4075        let response = ClientControlResponse::ServerDescribe {
4076            protocol_ver: PROTOCOL_VERSION,
4077            subc_ops: subc_ops(),
4078            capabilities: self.describe_capabilities(),
4079            connected_clients: self.connected_clients.count(),
4080            counters: Some(counters),
4081            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4082            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4083            capability_requirements: self.capability_requirement_statuses(),
4084            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4085            config_path: self
4086                .rescan
4087                .as_ref()
4088                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4089        };
4090        Ok(vec![control_response_body_frame(
4091            &frame,
4092            &response,
4093            "ClientControlResponse::ServerDescribe",
4094        )?])
4095    }
4096
4097    fn handle_catalog_list(
4098        &self,
4099        frame: Frame,
4100        module_id: Option<String>,
4101    ) -> Result<Vec<Frame>, RouterError> {
4102        let (generation, modules) = self
4103            .registry
4104            .list_modules_with_nonce_verification()
4105            .map_err(|err| {
4106                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4107            })?;
4108        let entries = modules
4109            .into_iter()
4110            .filter(|(registration, _)| {
4111                module_id
4112                    .as_deref()
4113                    .map(|wanted| registration.manifest.module_id == wanted)
4114                    .unwrap_or(true)
4115            })
4116            .map(|(registration, nonce_verified)| {
4117                let not_ready = self.not_ready_reason(&registration);
4118                let roles = registration.manifest.provides;
4119                CatalogEntry::new(
4120                    registration.manifest.module_id,
4121                    roles,
4122                    registration.control_ops,
4123                )
4124                .with_ready(not_ready.is_none())
4125                .with_not_ready(not_ready)
4126                .with_module_version(Some(registration.manifest.module_version))
4127                .with_capabilities(registration.manifest.capabilities)
4128                .with_self_signals(registration.manifest.self_signals)
4129                .with_events(registration.manifest.events)
4130                .with_operation_delivery(registration.manifest.operation_delivery)
4131                .with_operation_payload(registration.manifest.operation_payload)
4132                .with_flow_operations(registration.manifest.flow_operations)
4133                .with_confirm_operations(registration.manifest.confirm_operations)
4134                .with_rich_ui(registration.manifest.rich_ui)
4135                .with_role_declarations(registration.manifest.role_declarations)
4136                .with_nonce_verified(nonce_verified)
4137            })
4138            .collect();
4139        let response = ClientControlResponse::CatalogList {
4140            generation,
4141            modules: entries,
4142            subc_ops: subc_ops(),
4143        };
4144        Ok(vec![control_response_body_frame(
4145            &frame,
4146            &response,
4147            "ClientControlResponse::CatalogList",
4148        )?])
4149    }
4150
4151    /// The principal a route.open's `consumer_identity` attests: `Direct`
4152    /// when it presents none, `Reserved` when its launch nonce verifies, and
4153    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4154    /// can run before any refusal check without changing which check refuses.
4155    fn attest_route_open_consumer(
4156        &self,
4157        consumer_identity: Option<&ConsumerIdentity>,
4158    ) -> Option<Principal> {
4159        let Some(consumer_identity) = consumer_identity else {
4160            return Some(Principal::Direct);
4161        };
4162        self.supervisor
4163            .spawned_consumer_authorized(
4164                &consumer_identity.module_id,
4165                &consumer_identity.launch_nonce,
4166            )
4167            .then(|| Principal::Reserved {
4168                module_id: consumer_identity.module_id.clone(),
4169            })
4170    }
4171
4172    /// The caller fields for a refusal that only has the raw request frame,
4173    /// such as the connection-level admission refusal sent before the request
4174    /// reaches [`Self::handle_route_open`].
4175    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4176        let Ok(ClientControlRequest::RouteOpen {
4177            identity,
4178            consumer_identity,
4179            scope,
4180            ..
4181        }) = parse_client_control_request(&frame.body)
4182        else {
4183            return RouteOpenCaller::unknown();
4184        };
4185        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4186        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4187    }
4188
4189    fn route_open_principal(
4190        &self,
4191        frame: &Frame,
4192        consumer_identity: Option<ConsumerIdentity>,
4193    ) -> Result<Result<Principal, Frame>, RouterError> {
4194        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4195        self.route_open_principal_attested(frame, consumer_identity, attested)
4196    }
4197
4198    /// [`Self::route_open_principal`] with `attested` already taken by
4199    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4200    /// open checks the launch nonce once.
4201    fn route_open_principal_attested(
4202        &self,
4203        frame: &Frame,
4204        consumer_identity: Option<ConsumerIdentity>,
4205        attested: Option<Principal>,
4206    ) -> Result<Result<Principal, Frame>, RouterError> {
4207        if let Some(principal) = attested {
4208            return Ok(Ok(principal));
4209        }
4210        let Some(consumer_identity) = consumer_identity else {
4211            return Ok(Ok(Principal::Direct));
4212        };
4213
4214        Ok(Err(control_error_frame(
4215            frame,
4216            "bad_consumer_identity",
4217            format!(
4218                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4219                consumer_identity.module_id
4220            ),
4221        )?))
4222    }
4223
4224    /// Ordinary `route.open` refusals go through here; admission and breaker
4225    /// refusals log separately with their capacity or breaker state. The daemon can
4226    /// attest which code it sent: without the event, a client's "the daemon
4227    /// refused me" and the daemon's own view could only be reconciled by
4228    /// argument. Malformed input (`invalid_project_root`) does not come here;
4229    /// rejecting a request that was never a valid open is not a refusal of one.
4230    #[allow(clippy::too_many_arguments)]
4231    fn route_open_refusal_frame(
4232        &self,
4233        ctx: &RouteCtx,
4234        caller: &RouteOpenCaller,
4235        frame: &Frame,
4236        module_id: &str,
4237        reason: &'static str,
4238        code: &'static str,
4239        message: impl Into<String>,
4240    ) -> Result<Frame, RouterError> {
4241        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4242        control_error_frame(frame, code, message.into())
4243    }
4244
4245    /// Answer a waiting `route.open` whose relayed bind was refused.
4246    ///
4247    /// `daemon_authored` is provenance, not a code match: the daemon
4248    /// synthesized the refusal itself (a drain, a late reply, a scope that
4249    /// ended mid-bind) or the module refused the bind with its own body. A
4250    /// daemon refusal carries the daemon-origin bit; a module's own
4251    /// rejection is relayed with the bit clear, so a module that refuses
4252    /// with `module_reloading` still cannot mint a daemon refusal.
4253    #[allow(clippy::too_many_arguments)]
4254    fn route_bind_rejection(
4255        &self,
4256        ctx: &RouteCtx,
4257        caller: &RouteOpenCaller,
4258        frame: &Frame,
4259        target_module_id: &str,
4260        body: ErrorBody,
4261        daemon_authored: bool,
4262    ) -> Result<Vec<Frame>, RouterError> {
4263        // The daemon's own commit re-check refused the bind because the
4264        // scope ended or changed after admission. The module accepted;
4265        // counting it as a module rejection would blame the module. This
4266        // branch only runs for daemon-synthesized bodies: a module that
4267        // refuses with a scope code is answered as a module rejection below.
4268        if daemon_authored {
4269            let scope_code = match body.code.as_str() {
4270                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4271                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4272                _ => None,
4273            };
4274            if let Some(code) = scope_code {
4275                self.observe_route_open_refusal(
4276                    ctx,
4277                    caller,
4278                    target_module_id,
4279                    "scope_changed_before_commit",
4280                    code,
4281                );
4282                return Ok(vec![control_error_body_frame(frame, body)?]);
4283            }
4284        }
4285        self.counters
4286            .increment_route_open_refused("module_rejected");
4287        info!(
4288            target: "control",
4289            code = "module_rejected",
4290            module_code = ?body.code,
4291            module_id = ?target_module_id,
4292            connection_id = ctx.connection_id.get(),
4293            principal = %caller.principal,
4294            scope = %caller.scope,
4295            harness = %caller.harness,
4296            "route.open refused"
4297        );
4298        let refusal = if daemon_authored {
4299            control_error_body_frame(frame, body)?
4300        } else {
4301            control_relayed_error_body_frame(frame, body)?
4302        };
4303        Ok(vec![refusal])
4304    }
4305
4306    /// Refuse a `route.open` because the target module's bind-relay breaker is
4307    /// open, without attempting the relay.
4308    ///
4309    /// The wire code is `module_timeout`, which is the truth (the module has
4310    /// not been answering binds) and which both SDKs already classify as
4311    /// retryable with capped backoff. Reusing it is what keeps this change out
4312    /// of both SDKs; the daemon-side distinction lives in the counter key
4313    /// instead.
4314    ///
4315    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4316    /// While a breaker is open this fires on every open to that module, and the
4317    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4318    /// produced 261 lines about a single module inside 3000 lines of daemon
4319    /// log. The rare transitions are logged at warn/info instead and the volume
4320    /// is carried by the counter, so the evidence survives without the flood.
4321    /// The debug line keeps a per-refusal record reachable for whoever turns
4322    /// the level up.
4323    #[allow(clippy::too_many_arguments)]
4324    fn route_open_breaker_refusal_frame(
4325        &self,
4326        ctx: &RouteCtx,
4327        caller: &RouteOpenCaller,
4328        frame: &Frame,
4329        module_id: &str,
4330        consecutive_timeouts: u32,
4331        retry_in: Duration,
4332        probe_in_flight: bool,
4333    ) -> Result<Frame, RouterError> {
4334        self.counters
4335            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4336        debug!(
4337            target: "control",
4338            code = "module_timeout",
4339            module_id = ?module_id,
4340            connection_id = ctx.connection_id.get(),
4341            principal = %caller.principal,
4342            scope = %caller.scope,
4343            harness = %caller.harness,
4344            consecutive_timeouts,
4345            retry_in_ms = retry_in.as_millis() as u64,
4346            probe_in_flight,
4347            "route.open refused by open bind-relay breaker"
4348        );
4349        // Say what a caller can act on. An open bind-relay breaker means the
4350        // module timed out accepting several new routes in a row. The module
4351        // is still running and its established routes keep working; only new
4352        // route.open requests are refused until the cooldown ends and one
4353        // test route (the probe) gets through. A message that only counts
4354        // failed relays reads as "the module is down" to a worker that sees it.
4355        let detail = if probe_in_flight {
4356            "one test route is already being tried; retry once it settles".to_string()
4357        } else {
4358            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4359        };
4360        control_error_frame(
4361            frame,
4362            "module_timeout",
4363            format!(
4364                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4365                 timed out in a row); {detail}; its established routes are unaffected"
4366            ),
4367        )
4368    }
4369
4370    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4371    /// requester's bytes (an unknown target is whatever the client sent) and
4372    /// is Debug-formatted so control characters land in the log escaped
4373    /// rather than as terminal sequences for whoever tails it.
4374    ///
4375    /// `reason` names the check that refused, because one wire code has
4376    /// several senders: after a module registers, `target_unavailable` can
4377    /// come from a missing role, an inactive registration, a supervisor that
4378    /// has not marked the process live, a missing forwarding connection, or a
4379    /// failed relay, and a log that records only the code cannot say which of
4380    /// them fired. It is a static, daemon-chosen label per branch, so it is
4381    /// safe to print plainly and stays a closed set.
4382    fn observe_route_open_refusal(
4383        &self,
4384        ctx: &RouteCtx,
4385        caller: &RouteOpenCaller,
4386        module_id: &str,
4387        reason: &'static str,
4388        code: &'static str,
4389    ) {
4390        self.counters.increment_route_open_refused(code);
4391        info!(
4392            target: "control",
4393            code,
4394            reason,
4395            module_id = ?module_id,
4396            connection_id = ctx.connection_id.get(),
4397            principal = %caller.principal,
4398            scope = %caller.scope,
4399            harness = %caller.harness,
4400            "route.open refused"
4401        );
4402        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4403            self.route_outages.record_not_serving(module_id, reason);
4404        }
4405    }
4406
4407    /// Record an ACCEPTED route.open.
4408    ///
4409    /// Refusals have been logged and counted since the attestation work; accepts
4410    /// were invisible, so the daemon knew every principal it stamped and wrote
4411    /// none of them down. The party that attests the identity was the only party
4412    /// not recording it, which left a credential vault unable to name the sender
4413    /// of a call that reached it (claustrum #43) and left the launch-nonce
4414    /// concurrency question unanswerable from the outside.
4415    ///
4416    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4417    /// `code`/`module_id`/`connection_id` returns both directions of the same
4418    /// decision rather than two shapes a reader has to join by hand.
4419    ///
4420    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4421    /// and the difference carries information rather than being an
4422    /// inconsistency. This line is only reachable after a successful bind to a
4423    /// REGISTERED module, so the value has already passed HELLO validation
4424    /// including the path-hazard refusal and cannot contain control bytes. A
4425    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4426    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4427    ///
4428    /// Bare is also what every other daemon line already emits (`module
4429    /// registered`, `configured module supervised`). Shipping `?module_id` here
4430    /// made this instrument the only one in the file whose ids did not answer
4431    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4432    /// line whose whole purpose is being grepped beside its sibling.
4433    ///
4434    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4435    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4436    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4437    /// one are recorded identically. A test written against that harness passes
4438    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4439    /// green assertion that cannot fail. The same limit applies to the escaping
4440    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4441    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4442    /// Fencing either needs the real formatter, not the capture layer.
4443    ///
4444    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4445    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4446    /// to record. The ephemeral port would decay within minutes and answer only a
4447    /// live question. The identity question is instead answered by counting
4448    /// distinct live connections presenting one module's `consumer_identity` --
4449    /// "is anyone else holding this secret" rather than "is this the right
4450    /// process".
4451    fn observe_route_open_accept(
4452        &self,
4453        ctx: &RouteCtx,
4454        module_id: &str,
4455        principal: &str,
4456        scope: &str,
4457    ) {
4458        self.route_outages.record_accepted(module_id);
4459        self.counters.increment_route_open_accepted(principal);
4460        info!(
4461            target: "control",
4462            principal,
4463            module_id,
4464            connection_id = ctx.connection_id.get(),
4465            scope = %scope,
4466            "route.open accepted"
4467        );
4468    }
4469
4470    fn supervised_absent_route_open_refusal_frame(
4471        &self,
4472        ctx: &RouteCtx,
4473        caller: &RouteOpenCaller,
4474        frame: &Frame,
4475        module_id: &str,
4476        code: &'static str,
4477        status: &crate::supervise::ModuleStatus,
4478    ) -> Result<Frame, RouterError> {
4479        self.counters.increment_route_open_refused(code);
4480        info!(
4481            target: "control",
4482            code,
4483            reason = "supervised_not_registered",
4484            module_id = ?module_id,
4485            connection_id = ctx.connection_id.get(),
4486            principal = %caller.principal,
4487            scope = %caller.scope,
4488            harness = %caller.harness,
4489            state = %status.state,
4490            enabled = status.enabled,
4491            live = status.live,
4492            "route.open refused"
4493        );
4494        // A supervised module whose process has not registered is not
4495        // serving, whatever the reason; the supervisor knows this id, so it is
4496        // safe to track.
4497        self.route_outages
4498            .record_not_serving(module_id, "supervised_not_registered");
4499        let message = if let Some(pid) = status.waiting_for_previous_pid {
4500            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4501                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4502            })
4503        } else {
4504            format!(
4505                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4506                status.state, status.enabled, status.live
4507            )
4508        };
4509        control_error_frame(frame, code, message)
4510    }
4511
4512    async fn handle_route_open(
4513        &self,
4514        ctx: &RouteCtx,
4515        frame: Frame,
4516        request: RouteOpenRequest,
4517        timings: &mut RouteOpenTimings,
4518    ) -> Result<Vec<Frame>, RouterError> {
4519        let RouteOpenRequest {
4520            target,
4521            mut identity,
4522            consumer_identity,
4523            consumer_capabilities,
4524            role_versions,
4525            admission_facts,
4526            scope,
4527        } = request;
4528        let target_module_id = target_module_id(&target).to_string();
4529        timings.module_id.clone_from(&target_module_id);
4530        if self
4531            .registry
4532            .get_module_by_connection(ctx.connection_id)
4533            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4534            .is_some()
4535        {
4536            return Ok(vec![control_error_frame(
4537                &frame,
4538                "invalid_request",
4539                "module connections cannot open client routes",
4540            )?]);
4541        }
4542        debug!(
4543            connection_id = ctx.connection_id.get(),
4544            corr = frame.header.corr,
4545            module_id = %target_module_id,
4546            "handling route.open"
4547        );
4548
4549        // The caller's attested principal, requested scope and harness, for
4550        // every `route.open refused` line below. The launch nonce is checked
4551        // here, once, and the answer is reused where the principal is
4552        // enforced further down, so the order of refusals is unchanged.
4553        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4554        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4555
4556        // A malformed declaration is refused first, before anything about the
4557        // target is looked up: the same body would be refused against any
4558        // module, so the caller learns nothing by retrying or waiting. An empty
4559        // map declares nothing and travels as no field at all, so a provider
4560        // only ever sees a missing field or a non-empty one.
4561        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4562        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4563            self.observe_route_open_refusal(
4564                ctx,
4565                &caller,
4566                &target_module_id,
4567                "invalid_role_versions",
4568                error_codes::INVALID_REQUEST,
4569            );
4570            return Ok(vec![control_error_body_frame(
4571                &frame,
4572                ErrorBody {
4573                    code: error_codes::INVALID_REQUEST.to_string(),
4574                    message: error.to_string(),
4575                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4576                },
4577            )?]);
4578        }
4579
4580        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4581            if status.waiting_for_previous_pid.is_some() {
4582                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4583                    ctx,
4584                    &caller,
4585                    &frame,
4586                    &target_module_id,
4587                    "module_warming",
4588                    &status,
4589                )?]);
4590            }
4591        }
4592
4593        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4594        // opposite. Below, a caller learns whether a module is unregistered,
4595        // supervised-but-down (with state/enabled/live), or registered without the
4596        // requested role. Elsewhere that is an enumeration leak: a probe learning
4597        // the shape of a fleet it cannot otherwise see.
4598        //
4599        // It is not one here, and the reason is the ACCESS MODEL rather than
4600        // anything about these errors. Reaching route.open requires the
4601        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4602        // connection file, so any caller who completes it already runs as this
4603        // user -- and can read subc.jsonc for the module list and `ck module
4604        // status` for live state. The reply discloses nothing the caller cannot
4605        // read more easily from disk, while the precision is load-bearing:
4606        // `unknown_module` is retryable and a missing role is not.
4607        //
4608        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4609        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4610        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4611        let Some(registration) = self
4612            .registry
4613            .get_module(&target_module_id)
4614            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4615        else {
4616            if let Some((status, warming)) =
4617                self.supervisor_status(&target_module_id, frame.header.corr)?
4618            {
4619                // BEFORE the two availability codes below, because for a module
4620                // that speaks no subc wire both of them are false comfort: they
4621                // say "not right now" and are retried, and this module will
4622                // never register no matter how long the caller waits. The
4623                // absence here is the declaration being honoured, not a module
4624                // that is late.
4625                if status.protocol == ModuleProtocol::None {
4626                    return Ok(vec![self.route_open_refusal_frame(
4627                        ctx,
4628                        &caller,
4629                        &frame,
4630                        &target_module_id,
4631                        "protocol_none",
4632                        error_codes::MODULE_NO_PROTOCOL,
4633                        format!(
4634                            "module_id '{target_module_id}' is declared protocol: none; \
4635                             it speaks no subc wire and serves no routes"
4636                        ),
4637                    )?]);
4638                }
4639                let code = if warming {
4640                    "module_warming"
4641                } else {
4642                    "target_unavailable"
4643                };
4644                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4645                    ctx,
4646                    &caller,
4647                    &frame,
4648                    &target_module_id,
4649                    code,
4650                    &status,
4651                )?]);
4652            }
4653            if let Some(removed_ago_ms) =
4654                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4655            {
4656                return Ok(vec![self.route_open_refusal_frame(
4657                    ctx,
4658                    &caller,
4659                    &frame,
4660                    &target_module_id,
4661                    "removed",
4662                    error_codes::MODULE_REMOVED,
4663                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4664                )?]);
4665            }
4666            return Ok(vec![self.route_open_refusal_frame(
4667                ctx,
4668                &caller,
4669                &frame,
4670                &target_module_id,
4671                "not_registered",
4672                error_codes::UNKNOWN_MODULE,
4673                format!("module_id '{target_module_id}' is not registered"),
4674            )?]);
4675        };
4676
4677        // Best-effort only: registry readiness and forwarding reservation use
4678        // different locks, so a module can flip readiness between this read and
4679        // the relay. Modules must still tolerate an `on_bind` while not ready.
4680        if !registration.ready {
4681            self.counters
4682                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4683            info!(
4684                target: "control",
4685                code = error_codes::MODULE_WARMING,
4686                module_id = ?target_module_id,
4687                connection_id = ctx.connection_id.get(),
4688                principal = %caller.principal,
4689                scope = %caller.scope,
4690                harness = %caller.harness,
4691                reason = "declared_not_ready",
4692                "route.open refused"
4693            );
4694            // The module is registered but says it cannot take work, which is
4695            // an outage from the caller's side even though its process is up.
4696            self.route_outages
4697                .record_not_serving(&target_module_id, "declared_not_ready");
4698            return Ok(vec![control_error_body_frame(
4699                &frame,
4700                ErrorBody {
4701                    code: error_codes::MODULE_WARMING.to_string(),
4702                    message: format!(
4703                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4704                    ),
4705                    detail: Some(serde_json::json!({
4706                        "reason": "declared_not_ready"
4707                    })),
4708                },
4709            )?]);
4710        }
4711
4712        // Effective readiness, second half: a module that declares a capability
4713        // `need: required` is not routable while that capability has no
4714        // registered provider. It is enforced HERE, as a retryable routing
4715        // refusal, and deliberately not as spawn ordering or a boot block. The
4716        // module is still started and registered and can make its own calls;
4717        // spawn ordering is a promise that cannot be kept once a provider
4718        // crashes at runtime, and refusing to boot would stop the whole
4719        // machine, including the tools needed to fix its configuration.
4720        //
4721        // "Provided" is the evaluator's verdict, which counts a provider as
4722        // soon as it has REGISTERED, not once it is ready. Two modules that
4723        // require each other's capabilities are therefore both routable once
4724        // both register; counting readiness instead would deadlock them.
4725        //
4726        // Only new opens are refused. Routes already bound when a provider
4727        // goes away stay bound: nothing here tears them down, and the module
4728        // answers them as it can. Like the readiness read above this is
4729        // best-effort against a provider registering or leaving concurrently.
4730        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4731            self.counters
4732                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4733            info!(
4734                target: "control",
4735                code = error_codes::MODULE_WARMING,
4736                module_id = ?target_module_id,
4737                connection_id = ctx.connection_id.get(),
4738                principal = %caller.principal,
4739                scope = %caller.scope,
4740                harness = %caller.harness,
4741                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4742                capability = %capability,
4743                "route.open refused"
4744            );
4745            return Ok(vec![control_error_body_frame(
4746                &frame,
4747                ErrorBody {
4748                    code: error_codes::MODULE_WARMING.to_string(),
4749                    message: format!(
4750                        "module_id '{target_module_id}' requires capability '{capability}', \
4751                         which no registered module provides; retry"
4752                    ),
4753                    detail: Some(serde_json::json!({
4754                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4755                        "capability": capability,
4756                    })),
4757                },
4758            )?]);
4759        }
4760
4761        if !target_has_required_role(&target, &registration.manifest.provides) {
4762            return Ok(vec![self.route_open_refusal_frame(
4763                ctx,
4764                &caller,
4765                &frame,
4766                &target_module_id,
4767                "role_not_provided",
4768                "target_unavailable",
4769                format!("module_id '{target_module_id}' does not provide the requested target"),
4770            )?]);
4771        }
4772
4773        if registration.state != ChannelState::Active {
4774            return Ok(vec![self.route_open_refusal_frame(
4775                ctx,
4776                &caller,
4777                &frame,
4778                &target_module_id,
4779                "registration_not_active",
4780                "target_unavailable",
4781                format!("module_id '{target_module_id}' is not active"),
4782            )?]);
4783        }
4784
4785        if self
4786            .forwarding
4787            .module_is_draining(&target_module_id)
4788            .map_err(RouterError::Forwarding)?
4789        {
4790            return Ok(vec![self.route_open_refusal_frame(
4791                ctx,
4792                &caller,
4793                &frame,
4794                &target_module_id,
4795                "reloading",
4796                "module_reloading",
4797                format!("module_id '{target_module_id}' is reloading"),
4798            )?]);
4799        }
4800
4801        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4802            process_liveness.process_live(&target_module_id) == Some(false)
4803        }) {
4804            // A module the supervisor is restarting or reloading can still hold
4805            // a registration: the old process before its connection closes, or
4806            // a new one that registered while the supervisor was draining. The
4807            // forwarding table does not see that as draining, but the consumer
4808            // should still be told to retry soon, exactly as for the drain
4809            // above, rather than that the target is unavailable.
4810            if process_liveness.process_replacing(&target_module_id) {
4811                return Ok(vec![self.route_open_refusal_frame(
4812                    ctx,
4813                    &caller,
4814                    &frame,
4815                    &target_module_id,
4816                    "reloading",
4817                    "module_reloading",
4818                    format!("module_id '{target_module_id}' is reloading"),
4819                )?]);
4820            }
4821            return Ok(vec![self.route_open_refusal_frame(
4822                ctx,
4823                &caller,
4824                &frame,
4825                &target_module_id,
4826                "supervisor_not_live",
4827                "target_unavailable",
4828                format!("module_id '{target_module_id}' is not live"),
4829            )?]);
4830        }
4831
4832        if !self
4833            .forwarding
4834            .has_live_module_connection(&target_module_id)
4835            .map_err(RouterError::Forwarding)?
4836        {
4837            return Ok(vec![self.route_open_refusal_frame(
4838                ctx,
4839                &caller,
4840                &frame,
4841                &target_module_id,
4842                "no_forwarding_connection",
4843                "target_unavailable",
4844                format!("module_id '{target_module_id}' has no live forwarding connection"),
4845            )?]);
4846        }
4847
4848        if let Some(error) =
4849            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4850        {
4851            self.observe_route_open_refusal(
4852                ctx,
4853                &caller,
4854                &target_module_id,
4855                "op_not_allowed",
4856                "op_not_allowed",
4857            );
4858            return Ok(vec![error]);
4859        }
4860
4861        let principal =
4862            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4863                Ok(principal) => principal,
4864                Err(error) => {
4865                    self.observe_route_open_refusal(
4866                        ctx,
4867                        &caller,
4868                        &target_module_id,
4869                        "bad_consumer_identity",
4870                        "bad_consumer_identity",
4871                    );
4872                    return Ok(vec![error]);
4873                }
4874            };
4875        timings.principal = principal_label(&principal);
4876
4877        // Only Callosum can attest a production phone build. Consume the claim
4878        // here so it can be kept on the route without reaching any module.
4879        let build_class = identity.build_class.take().filter(|build_class| {
4880            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
4881                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
4882            } else {
4883                if self
4884                    .build_class_warned
4885                    .lock()
4886                    .unwrap_or_else(|p| p.into_inner())
4887                    .insert(ctx.connection_id)
4888                {
4889                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
4890                        "ignoring build_class from a non-Callosum opener");
4891                }
4892                false
4893            }
4894        });
4895
4896        // This is attested, control-plane policy for supervised module origins.
4897        // Keep it before route reservation and out of the opaque forwarding hot
4898        // path: data frames must never acquire a per-frame capability check.
4899        if let Principal::Reserved {
4900            module_id: opening_module_id,
4901        } = &principal
4902        {
4903            if let Some(opening_registration) = self
4904                .registry
4905                .get_module(opening_module_id)
4906                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4907            {
4908                if let Some(capability) =
4909                    denied_capability(&opening_registration.manifest, &registration.manifest)
4910                {
4911                    warn!(
4912                        opening_module_id,
4913                        target_module_id,
4914                        capability,
4915                        "refusing route.open because an attested capability deny edge matches"
4916                    );
4917                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4918                        &frame,
4919                        &target_module_id,
4920                        "capability_deny_edge",
4921                        "capability_forbidden",
4922                        format!(
4923                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4924                        ),
4925                    )?]);
4926                }
4927            }
4928        }
4929
4930        if admission_facts.is_some() {
4931            let carrier_matches = matches!(
4932                &principal,
4933                Principal::Reserved { module_id }
4934                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4935            );
4936            if !carrier_matches {
4937                return Ok(vec![self.route_open_refusal_frame(
4938                    ctx,
4939                    &caller,
4940                    &frame,
4941                    &target_module_id,
4942                    "admission_facts_carrier_not_permitted",
4943                    "admission_facts_not_permitted",
4944                    "admission facts may only be carried by the configured reserved module",
4945                )?]);
4946            }
4947
4948            let target_allowed = self
4949                .admission_facts_targets
4950                .as_ref()
4951                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4952            if !target_allowed {
4953                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4954                    &frame,
4955                    &target_module_id,
4956                    "admission_facts_target_not_listed",
4957                    "admission_facts_target_not_allowed",
4958                    format!(
4959                        "admission facts are not permitted for target module_id '{target_module_id}'"
4960                    ),
4961                )?]);
4962            }
4963
4964            // Keep the value opaque to subc. The downstream admission validator owns
4965            // schema and semantic checks; this daemon only enforces carrier authority
4966            // and the configured destination allowlist.
4967        }
4968
4969        // Scope admission, on the attested principal above and never on the
4970        // request body. The tag read here travels with the pending bind and is
4971        // compared with the published one at commit, so a sync between here
4972        // and the module's ack refuses the open instead of binding a stamp
4973        // that is no longer true.
4974        let (bound_scope, mut scope_stamp) = match &scope {
4975            None => (None, None),
4976            Some(selector) => {
4977                let owner_configured = match &selector.owner {
4978                    // A reserved owner counts as configured from before its
4979                    // process is spawned (see `SupervisorHandle::is_configured`).
4980                    // So an owner that has not synced its scopes yet is refused
4981                    // as retryable (`scope_not_synced`), not as one that will
4982                    // never sync.
4983                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4984                    _ => false,
4985                };
4986                let mut scopes = self
4987                    .scopes
4988                    .write()
4989                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4990                let admitted = scopes
4991                    .admit(&principal, &target_module_id, selector, owner_configured)
4992                    .and_then(|admission| {
4993                        crate::scopes::check_target_flow_support(
4994                            &admission.stamp,
4995                            &target_module_id,
4996                            registration.manifest.capabilities.as_ref(),
4997                        )?;
4998                        crate::scopes::check_target_agent_run_support(
4999                            &admission.stamp,
5000                            &target_module_id,
5001                            registration.manifest.capabilities.as_ref(),
5002                        )?;
5003                        let bound = scopes.reserve_bind(
5004                            &admission,
5005                            &principal,
5006                            ctx.connection_id,
5007                            selector.hold_presence,
5008                        );
5009                        Ok((admission, bound))
5010                    });
5011                match admitted {
5012                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5013                    Err(refusal) => {
5014                        return Ok(vec![self.route_open_refusal_frame(
5015                            ctx,
5016                            &caller,
5017                            &frame,
5018                            &target_module_id,
5019                            refusal.code,
5020                            refusal.code,
5021                            refusal.message,
5022                        )?]);
5023                    }
5024                }
5025            }
5026        };
5027        // From here on a refusal names the scope by the stamp admission
5028        // issued, exactly as the `route.open accepted` line does.
5029        if let Some(stamp) = &scope_stamp {
5030            caller.scope = route_open_scope_label(stamp);
5031        }
5032
5033        // Bind admits a root that no longer exists on disk, because refusing here
5034        // closes the only exit from a paused run: cancel needs a bound route, and a
5035        // renamed or reclaimed directory makes that route unopenable forever. The
5036        // run itself is intact and still addressable by its recorded identity.
5037        //
5038        // This does NOT relax the rule the strict constructor protects. That rule is
5039        // that no root is ever aliased into NEW durable state -- a missing component
5040        // can reappear as a symlink elsewhere, which would move the identity and
5041        // split a session's history across two of them. The engine now refuses the
5042        // two operations that create such state (send and import) at admission,
5043        // which is a narrower way to hold the same invariant: reads and terminations
5044        // are admitted, writes are not. That refusal had to ship before this line
5045        // changed, or there is an interval where a send commits under a provisional
5046        // identity -- the exact failure the original policy existed to prevent.
5047        //
5048        // Resolution follows realpath rather than lexical cleanup: the longest
5049        // existing ancestor is canonicalized and the missing tail re-appended, so a
5050        // live root is unchanged and a vanished leaf keeps the identity it was
5051        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
5052        // same caller the moment the directory vanished, which strands the run more
5053        // quietly than refusing it.
5054        timings.enter(RouteOpenPhase::ProjectRoot);
5055        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
5056            Ok(project_root) => project_root,
5057            Err(err) => {
5058                return Ok(vec![control_error_frame(
5059                    &frame,
5060                    "invalid_project_root",
5061                    err.to_string(),
5062                )?])
5063            }
5064        };
5065        timings.enter(RouteOpenPhase::Admission);
5066        identity.project_root = project_root.as_path().to_path_buf();
5067
5068        // Last gate before any relay work, and deliberately after the cheap
5069        // registry and availability checks above: those name a more precise
5070        // condition (unknown, removed, reloading) and a caller is better served
5071        // by the precise code than by this one.
5072        //
5073        // Everything below this point costs an egress permit, a reserved handle
5074        // pair and, if the module does not answer, the whole relay budget. The
5075        // reader no longer waits for that budget, so cap each target explicitly;
5076        // serial dispatch used to provide the accidental cap of one relay per
5077        // connection. Admission is a mutex-protected count and never waits.
5078        let _concurrency_guard = match self
5079            .route_bind_concurrency
5080            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5081        {
5082            Ok(guard) => guard,
5083            Err(in_flight) => {
5084                return Ok(vec![self.route_open_target_capacity_refusal(
5085                    ctx,
5086                    &caller,
5087                    &frame,
5088                    &target_module_id,
5089                    in_flight,
5090                )?]);
5091            }
5092        };
5093
5094        // A module that has already burned the whole budget `threshold` times
5095        // in a row does not get to charge it again until a probe says it recovered.
5096        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5097            RouteBindAdmission::Admitted { guard, probe } => {
5098                if probe {
5099                    info!(
5100                        module_id = %target_module_id,
5101                        connection_id = ctx.connection_id.get(),
5102                        "route.bind breaker half-open: admitting one probe"
5103                    );
5104                }
5105                guard
5106            }
5107            RouteBindAdmission::Refused {
5108                consecutive_timeouts,
5109                retry_in,
5110                probe_in_flight,
5111            } => {
5112                return Ok(vec![self.route_open_breaker_refusal_frame(
5113                    ctx,
5114                    &caller,
5115                    &frame,
5116                    &target_module_id,
5117                    consecutive_timeouts,
5118                    retry_in,
5119                    probe_in_flight,
5120                )?]);
5121            }
5122        };
5123
5124        // Resolve the per-module budget here so the wait matches the operator's
5125        // intent for this specific target. A per-module override in
5126        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5127        // daemons) wins over the daemon-wide default.
5128        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5129        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5130        let operator_app = self
5131            .operator_apps
5132            .lock()
5133            .unwrap_or_else(|p| p.into_inner())
5134            .get(&ctx.connection_id)
5135            .cloned();
5136        timings.enter(RouteOpenPhase::EgressReservation);
5137        let pending = match self
5138            .forwarding
5139            .begin_route_bind_relay_for(
5140                ctx.connection_id,
5141                ctx.egress.clone(),
5142                response_version(&frame),
5143                frame.header.corr,
5144                &target_module_id,
5145                principal.clone(),
5146                bound_scope,
5147                Some(project_root),
5148                identity.harness.clone(),
5149                build_class,
5150                operator_app.clone(),
5151                relay_deadline,
5152            )
5153            .await
5154        {
5155            Ok(pending) => pending,
5156            Err(err) => {
5157                return Ok(vec![self.route_open_refusal_frame(
5158                    ctx,
5159                    &caller,
5160                    &frame,
5161                    &target_module_id,
5162                    "relay_reservation_failed",
5163                    forwarding_error_code(&err),
5164                    err.to_string(),
5165                )?])
5166            }
5167        };
5168        timings.enter(RouteOpenPhase::Admission);
5169        let crate::forwarding::PendingRouteBindRelay {
5170            endpoint,
5171            module_sink,
5172            negotiated_ver,
5173            client_channel,
5174            client_epoch,
5175            module_channel,
5176            module_epoch,
5177            corr: relay_corr,
5178            receiver,
5179        } = pending;
5180        let mut reservation =
5181            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5182
5183        // Reserving egress can wait while a module reconnects or a swap cuts
5184        // over. Check the relay's captured connection, not the earlier by-id
5185        // lookup: the original module's flow/run capabilities cannot authorize
5186        // its replacement. The captured sink stays bound to that connection.
5187        let relay_registration = self
5188            .registry
5189            .get_module_by_connection(endpoint.connection_id)
5190            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5191        let relay_capabilities = relay_registration
5192            .as_ref()
5193            .and_then(|registration| registration.manifest.capabilities.as_ref());
5194        if let Some(selector) = &scope {
5195            // Egress reservation may wait across a scope update. Recheck the
5196            // proof against the current record before sending the captured bind.
5197            if let Err(refusal) = self
5198                .scopes
5199                .read()
5200                .unwrap_or_else(|poisoned| poisoned.into_inner())
5201                .check_bind_proof(&principal, selector)
5202            {
5203                reservation.release_and_disarm();
5204                return Ok(vec![self.route_open_refusal_frame(
5205                    ctx,
5206                    &caller,
5207                    &frame,
5208                    &target_module_id,
5209                    refusal.code,
5210                    refusal.code,
5211                    refusal.message,
5212                )?]);
5213            }
5214        }
5215        if let Some(stamp) = scope_stamp
5216            .as_ref()
5217            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5218        {
5219            if let Err(refusal) = crate::scopes::check_target_flow_support(
5220                stamp,
5221                &target_module_id,
5222                relay_capabilities,
5223            )
5224            .and_then(|()| {
5225                crate::scopes::check_target_agent_run_support(
5226                    stamp,
5227                    &target_module_id,
5228                    relay_capabilities,
5229                )
5230            }) {
5231                reservation.release_and_disarm();
5232                return Ok(vec![self.route_open_refusal_frame(
5233                    ctx,
5234                    &caller,
5235                    &frame,
5236                    &target_module_id,
5237                    refusal.code,
5238                    refusal.code,
5239                    refusal.message,
5240                )?]);
5241            }
5242        }
5243
5244        // Older readers may reject unknown fields. Select optional fields from
5245        // the captured endpoint's declaration, including on unscoped binds.
5246        if !relay_capabilities.is_some_and(|caps| {
5247            caps.provides
5248                .iter()
5249                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5250        }) {
5251            if let Some(stamp) = scope_stamp.as_mut() {
5252                stamp.workdir = None;
5253                stamp.project_id = None;
5254            }
5255        }
5256        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5257        // attribution. The run gate above has already required the target to
5258        // declare `worker-runs/v1`, which promises to read both
5259        // `on_behalf_of_agent` and `call_mode`.
5260        let worker_run = scope_stamp
5261            .as_ref()
5262            .is_some_and(crate::scopes::is_worker_run)
5263            && relay_capabilities.is_some_and(|caps| {
5264                caps.provides
5265                    .iter()
5266                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5267            });
5268        if !worker_run
5269            && !relay_capabilities.is_some_and(|caps| {
5270                caps.provides
5271                    .iter()
5272                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5273            })
5274        {
5275            if let Some(stamp) = scope_stamp.as_mut() {
5276                stamp.attributes.on_behalf_of_agent = None;
5277            }
5278        }
5279        if !relay_capabilities.is_some_and(|caps| {
5280            caps.provides
5281                .iter()
5282                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5283        }) {
5284            if let Some(stamp) = scope_stamp.as_mut() {
5285                stamp.attributes.flow_version = None;
5286                stamp.attributes.flow_code_hash = None;
5287                stamp.attributes.flow_install_id = None;
5288            }
5289        }
5290        let call_mode = if worker_run
5291            || relay_capabilities
5292                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5293        {
5294            Some(derive_call_mode(scope_stamp.as_ref()))
5295        } else {
5296            if let Some(stamp) = scope_stamp.as_mut() {
5297                stamp.expires_at_ms = None;
5298            }
5299            None
5300        };
5301
5302        debug!(
5303            connection_id = ctx.connection_id.get(),
5304            client_channel,
5305            client_epoch,
5306            module_channel,
5307            module_epoch,
5308            "reserved route handle pair"
5309        );
5310        // Rendered BEFORE the move into the relay, because the accept arm below
5311        // is where it is logged and the principal is gone by then.
5312        let route_principal_label = principal_label(&principal);
5313        let relay = ModuleControlRequest::RouteBind {
5314            route_channel: module_channel,
5315            epoch: module_epoch,
5316            target,
5317            identity,
5318            principal: Some(principal),
5319            consumer_capabilities,
5320            role_versions,
5321            admission_facts,
5322            scope: scope_stamp,
5323            call_mode,
5324        };
5325        let peer = RouteBindPeer::new()
5326            .with_observed_peer(
5327                relay_capabilities
5328                    .is_some_and(|caps| {
5329                        caps.provides
5330                            .iter()
5331                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5332                    })
5333                    .then(|| {
5334                        self.observed_peers
5335                            .lock()
5336                            .unwrap_or_else(|p| p.into_inner())
5337                            .get(&ctx.connection_id)
5338                            .cloned()
5339                            .unwrap_or_else(ObservedPeer::unobserved)
5340                    }),
5341            )
5342            .with_operator_app(
5343                operator_app
5344                    .filter(|_| {
5345                        self.operator_app_policy.macos_enabled
5346                            && relay_capabilities.is_some_and(|caps| {
5347                                caps.provides
5348                                    .iter()
5349                                    .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5350                            })
5351                    })
5352                    .map(|stamp| stamp.app),
5353            );
5354        let relay_body = encode_route_bind(&relay, &peer).map_err(|err| {
5355            RouterError::backend(
5356                0,
5357                frame.header.corr,
5358                format!("failed to encode route.bind request: {err}"),
5359            )
5360        })?;
5361        let relay_frame = Frame::build_with_version(
5362            negotiated_ver,
5363            FrameType::Request,
5364            control_flags(),
5365            0,
5366            0,
5367            relay_corr,
5368            relay_body,
5369        )
5370        .map_err(RouterError::FrameBuild)?;
5371
5372        timings.enter(RouteOpenPhase::BindSend);
5373        let sent = module_sink.send(relay_frame).await;
5374        timings.enter(RouteOpenPhase::Admission);
5375        if let Err(err) = sent {
5376            reservation.release_and_disarm();
5377            return Ok(vec![self.route_open_refusal_frame(
5378                ctx,
5379                &caller,
5380                &frame,
5381                &target_module_id,
5382                "relay_send_failed",
5383                "target_unavailable",
5384                err.to_string(),
5385            )?]);
5386        }
5387
5388        #[cfg(test)]
5389        if let Some(hold) = &self.route_bind_relay_sent_hold {
5390            // First wait: tell the test the bind is queued. Second wait: stay
5391            // here until the test has driven whatever it wants to happen
5392            // between queueing the bind and marking it enqueued.
5393            hold.wait().await;
5394            hold.wait().await;
5395        }
5396
5397        // A missing relay is not by itself an abandoned bind: the module may
5398        // already have acked it and the route may be committed. Only a relay
5399        // dropped without any module reply needs this GOODBYE; one a reply
5400        // settled has already had its GOODBYE decided by the reply path.
5401        if self
5402            .forwarding
5403            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5404            .map_err(RouterError::Forwarding)?
5405            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5406        {
5407            self.send_abandoned_route_bind_goodbye(
5408                &module_sink,
5409                negotiated_ver,
5410                module_channel,
5411                module_epoch,
5412            );
5413        }
5414
5415        timings.enter(RouteOpenPhase::BindAckWait);
5416        let outcome = timeout_at(relay_deadline, receiver).await;
5417        timings.enter(RouteOpenPhase::Admission);
5418        match outcome {
5419            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5420                reservation.disarm();
5421                if breaker.record_accepted() {
5422                    info!(
5423                        module_id = %target_module_id,
5424                        "route.bind breaker closed: the probe was accepted"
5425                    );
5426                }
5427                self.observe_route_open_accept(
5428                    ctx,
5429                    &target_module_id,
5430                    &route_principal_label,
5431                    &caller.scope,
5432                );
5433                Ok(Vec::new())
5434            }
5435            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5436                reservation.release_and_disarm();
5437                // A module that says no in microseconds is healthy. Rejection
5438                // is a different condition with its own refusal and must not
5439                // move the breaker.
5440                breaker.record_inconclusive();
5441                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5442            }
5443            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5444                reservation.release_and_disarm();
5445                breaker.record_inconclusive();
5446                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5447            }
5448            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5449                reservation.release_and_disarm();
5450                breaker.record_inconclusive();
5451                // Fires when the module's connection closes while a relayed
5452                // bind is pending -- typically a caller racing a module restart
5453                // whose bind was relayed BEFORE the drain mark went up. Logged
5454                // because the caller sees only its own error and the fleet has
5455                // already spent one diagnosis round unable to tell this arm
5456                // from a relay timeout without daemon-side evidence.
5457                tracing::warn!(
5458                    module_id = %target_module_id,
5459                    "route.bind relay abandoned: {message}"
5460                );
5461                Ok(vec![self.route_open_refusal_frame(
5462                    ctx,
5463                    &caller,
5464                    &frame,
5465                    &target_module_id,
5466                    "relay_abandoned",
5467                    "target_unavailable",
5468                    message,
5469                )?])
5470            }
5471            Ok(Err(_)) => {
5472                reservation.release_and_disarm();
5473                breaker.record_inconclusive();
5474                Ok(vec![self.route_open_refusal_frame(
5475                    ctx,
5476                    &caller,
5477                    &frame,
5478                    &target_module_id,
5479                    "relay_waiter_canceled",
5480                    "target_unavailable",
5481                    "route.bind relay waiter was canceled before the module responded",
5482                )?])
5483            }
5484            Err(_) => {
5485                reservation.release_and_disarm();
5486                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5487                // answer at all is the one condition a fast refusal can
5488                // usefully stand in for; every other arm already answered.
5489                if let Some(opened) = breaker.record_timeout(
5490                    self.route_bind_breaker_threshold,
5491                    self.route_bind_breaker_cooldown,
5492                ) {
5493                    warn!(
5494                        module_id = %target_module_id,
5495                        consecutive_timeouts = opened.consecutive_timeouts,
5496                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5497                        reopened_after_probe = opened.reopened_after_probe,
5498                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5499                    );
5500                }
5501                // The generous budget just burned to no answer: the module is
5502                // registered and its connection is up, but its bind handler sat
5503                // on the ack for the full budget (warm-on-bind, cold configure,
5504                // or a wedged handler). Every earlier unavailability shape
5505                // fast-refuses BEFORE the relay, so this arm firing means the
5506                // slowness is module-side -- log it so the per-module timeline
5507                // is reconstructable without client audit rows.
5508                tracing::warn!(
5509                    module_id = %target_module_id,
5510                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5511                    "route.bind relay timed out: module did not ack within budget"
5512                );
5513                Ok(vec![self.route_open_refusal_frame(
5514                    ctx,
5515                    &caller,
5516                    &frame,
5517                    &target_module_id,
5518                    "relay_timed_out",
5519                    "module_timeout",
5520                    format!(
5521                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5522                        route_bind_relay_timeout
5523                    ),
5524                )?])
5525            }
5526        }
5527    }
5528
5529    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5530        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5531            snapshot: self.supervisor.spawn_snapshot(),
5532        };
5533        Ok(vec![control_response_body_frame(
5534            &frame,
5535            &response,
5536            "ClientControlResponse::SupervisorSpawnSnapshot",
5537        )?])
5538    }
5539
5540    fn handle_supervisor_spawn_subscribe(
5541        &self,
5542        ctx: &RouteCtx,
5543        frame: Frame,
5544        since: Option<SpawnCursor>,
5545    ) -> Result<Vec<Frame>, RouterError> {
5546        match self.supervisor.subscribe_spawns(
5547            ctx.connection_id,
5548            frame.header.corr,
5549            response_version(&frame),
5550            since,
5551            ctx.egress.clone(),
5552        ) {
5553            Ok(()) => Ok(Vec::new()),
5554            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5555                Ok(vec![control_error_body_frame(
5556                    &frame,
5557                    ErrorBody {
5558                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5559                        message: "spawn cursor belongs to a different daemon incarnation"
5560                            .to_string(),
5561                        detail: Some(serde_json::json!({
5562                            "current_daemon_incarnation": current
5563                        })),
5564                    },
5565                )?])
5566            }
5567            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5568                &frame,
5569                ErrorBody {
5570                    code: "spawn_cursor_too_old".to_string(),
5571                    message: "spawn cursor predates the retained event ring".to_string(),
5572                    detail: Some(serde_json::json!({
5573                        "oldest_retained_cursor": oldest
5574                    })),
5575                },
5576            )?]),
5577            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5578                0,
5579                frame.header.corr,
5580                format!("failed to open supervisor spawn subscription: {error}"),
5581            )),
5582        }
5583    }
5584
5585    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5586        let generation = self
5587            .registry
5588            .generation()
5589            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5590        let mut modules = Vec::new();
5591        for module in self.supervisor.list() {
5592            let status = module.status_for_control("list").map_err(|err| {
5593                RouterError::backend(
5594                    0,
5595                    frame.header.corr,
5596                    format!("failed to read supervisor status: {err}"),
5597                )
5598            })?;
5599            let (configured, _) = module.configuration().map_err(|err| {
5600                RouterError::backend(
5601                    0,
5602                    frame.header.corr,
5603                    format!("failed to read module configuration: {err}"),
5604                )
5605            })?;
5606            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5607                RouterError::backend(
5608                    0,
5609                    frame.header.corr,
5610                    format!("failed to read retry time: {err}"),
5611                )
5612            })?;
5613            // Status and configuration snapshots release their locks before the image probe awaits.
5614            let image = module.running_image_agreement().await;
5615            // Read per request so the figure is current when the operator asks;
5616            // the daemon samples nothing in between.
5617            let resources = Some(module.child_resource_usage());
5618            let mut verdict =
5619                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5620            verdict.spec = Some(launch_spec_agreement(
5621                &configured,
5622                status.process_alive,
5623                status.launch_spec_digest.as_ref(),
5624            ));
5625            let pending_reload = Some(verdict);
5626            let configured_enabled = self
5627                .capability_evaluator
5628                .configured_enabled(&status.module_id);
5629            // Keep the retired policy field on the wire for one release so
5630            // existing status consumers still receive the platform policy.
5631            modules.push(
5632                SupervisorEntry::new(
5633                    status.module_id,
5634                    status.state.to_string(),
5635                    status.enabled,
5636                    status.live,
5637                    status.health.status,
5638                )
5639                .with_configured_enabled(configured_enabled)
5640                .with_next_retry_at_ms(next_retry_at_ms)
5641                .with_launch_nonce_env(Some(!cfg!(unix)))
5642                .with_launch_nonce_source(status.launch_nonce_source)
5643                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5644                .with_protocol(status.protocol)
5645                .with_pending_reload(pending_reload)
5646                .with_last_probe_ms(status.health.last_probe_ms)
5647                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5648                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5649                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5650                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5651                .with_restart_count(Some(status.restart_count))
5652                .with_max_restarts(Some(status.max_restarts))
5653                .with_lifetime_restarts(Some(status.lifetime_restarts))
5654                .with_spawn_generation(Some(status.spawn_generation))
5655                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5656                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5657                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5658                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5659                .with_resources(resources),
5660            );
5661        }
5662        let response = ClientControlResponse::SupervisorList {
5663            generation,
5664            modules,
5665        };
5666        Ok(vec![control_response_body_frame(
5667            &frame,
5668            &response,
5669            "ClientControlResponse::SupervisorList",
5670        )?])
5671    }
5672
5673    fn handle_supervisor_stderr_tail(
5674        &self,
5675        frame: Frame,
5676        module_id: String,
5677        max_lines: Option<u32>,
5678        max_bytes: Option<u32>,
5679    ) -> Result<Vec<Frame>, RouterError> {
5680        let Some(module) = self.supervisor.get(&module_id) else {
5681            return Ok(vec![control_error_frame(
5682                &frame,
5683                "unknown_module",
5684                format!("module_id '{module_id}' is not supervised"),
5685            )?]);
5686        };
5687
5688        let snapshot = module.stderr_tail(
5689            max_lines.map(|value| value as usize),
5690            max_bytes.map(|value| value as usize),
5691        );
5692
5693        let response = ClientControlResponse::SupervisorStderrTail {
5694            module_id,
5695            tail: StderrTail {
5696                capture: match snapshot.capture {
5697                    CaptureState::Captured => StderrCaptureState::Captured,
5698                    CaptureState::Incomplete { reason } => {
5699                        StderrCaptureState::Incomplete { reason }
5700                    }
5701                    CaptureState::NotCaptured { reason } => {
5702                        StderrCaptureState::NotCaptured { reason }
5703                    }
5704                },
5705                entries: snapshot
5706                    .entries
5707                    .into_iter()
5708                    .map(|entry| match entry {
5709                        TailEntry::Line {
5710                            text,
5711                            truncated,
5712                            at_ms,
5713                        } => StderrTailEntry::Line {
5714                            text,
5715                            truncated,
5716                            at_ms,
5717                        },
5718                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5719                    })
5720                    .collect(),
5721                dropped_lines: snapshot.dropped_lines,
5722            },
5723        };
5724        Ok(vec![control_response_body_frame(
5725            &frame,
5726            &response,
5727            "ClientControlResponse::SupervisorStderrTail",
5728        )?])
5729    }
5730
5731    async fn handle_supervisor_terminals(
5732        &self,
5733        frame: Frame,
5734        module_id: String,
5735    ) -> Result<Vec<Frame>, RouterError> {
5736        let Some(module) = self.supervisor.get(&module_id) else {
5737            return Ok(vec![control_error_frame(
5738                &frame,
5739                "unknown_module",
5740                format!("module_id '{module_id}' is not supervised"),
5741            )?]);
5742        };
5743
5744        // The journal read runs on a blocking thread: it can be megabytes of
5745        // file I/O and must not occupy a runtime worker.
5746        let terminals = module
5747            .read_durable_terminal_history()
5748            .await
5749            .map_err(|error| {
5750                RouterError::backend(
5751                    0,
5752                    frame.header.corr,
5753                    format!("failed to read terminal history: {error}"),
5754                )
5755            })?;
5756        let response = ClientControlResponse::SupervisorTerminals {
5757            module_id,
5758            terminals,
5759        };
5760        Ok(vec![control_response_body_frame(
5761            &frame,
5762            &response,
5763            "ClientControlResponse::SupervisorTerminals",
5764        )?])
5765    }
5766
5767    fn handle_supervisor_routes(
5768        &self,
5769        frame: Frame,
5770        module_id: Option<String>,
5771    ) -> Result<Vec<Frame>, RouterError> {
5772        let modules = self
5773            .forwarding
5774            .route_census(module_id.as_deref())
5775            .map_err(RouterError::Forwarding)?
5776            .into_iter()
5777            .map(|(module_id, routes)| SupervisorRouteModule {
5778                module_id,
5779                routes: routes
5780                    .into_iter()
5781                    .map(|route| SupervisorRoute {
5782                        consumer: match route.principal {
5783                            Principal::Reserved { module_id } => {
5784                                SupervisorRouteConsumer::Reserved { module_id }
5785                            }
5786                            Principal::Direct | Principal::Unverified => {
5787                                SupervisorRouteConsumer::Direct {
5788                                    connection_id: route.goodbye_target.connection_id.get(),
5789                                }
5790                            }
5791                        },
5792                        age_ms: Instant::now()
5793                            .saturating_duration_since(route.bound_at)
5794                            .as_millis()
5795                            .try_into()
5796                            .unwrap_or(u64::MAX),
5797                        draining: route.draining,
5798                        drain_reason: route.drain_reason,
5799                    })
5800                    .collect(),
5801            })
5802            .collect();
5803        let response = ClientControlResponse::SupervisorRoutes { modules };
5804        Ok(vec![control_response_body_frame(
5805            &frame,
5806            &response,
5807            "ClientControlResponse::SupervisorRoutes",
5808        )?])
5809    }
5810
5811    async fn handle_supervisor_provenance(
5812        &self,
5813        frame: Frame,
5814        module_id: Option<String>,
5815    ) -> Result<Vec<Frame>, RouterError> {
5816        let mut selected = if let Some(module_id) = module_id {
5817            let Some(module) = self.supervisor.get(&module_id) else {
5818                return Ok(vec![control_error_frame(
5819                    &frame,
5820                    "unknown_module",
5821                    format!("module_id '{module_id}' is not supervised"),
5822                )?]);
5823            };
5824            vec![module]
5825        } else {
5826            self.supervisor.list()
5827        };
5828
5829        let mut modules = Vec::with_capacity(selected.len());
5830        for module in selected.drain(..) {
5831            let (status, observed_image) = module
5832                .status_and_running_image_agreement()
5833                .await
5834                .map_err(|err| {
5835                    RouterError::backend(
5836                        0,
5837                        frame.header.corr,
5838                        format!("failed to read supervisor status: {err}"),
5839                    )
5840                })?;
5841            let module_declared = self
5842                .registry
5843                .get_module(&status.module_id)
5844                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5845                .and_then(|registration| registration.manifest.provenance)
5846                .map(|build| ModuleDeclaredProvenance::Reported { build })
5847                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5848            #[cfg(test)]
5849            let running_image = match &self.provenance_probe_override {
5850                Some(result) => result.clone(),
5851                None => observed_image,
5852            };
5853            #[cfg(not(test))]
5854            let running_image = observed_image;
5855            modules.push(SupervisorModuleProvenance {
5856                module_id: status.module_id,
5857                module_declared,
5858                daemon_observed: SupervisorObservedProcess {
5859                    launch_nonce_source: status.launch_nonce_source,
5860                    pid: status.pid,
5861                    spawned_at_ms: status.spawned_at_ms,
5862                    spawned_from: status.spawned_from,
5863                    running_image,
5864                },
5865            });
5866        }
5867        let daemon = SupervisorDaemonProvenance {
5868            daemon_build: self.daemon_provenance.build.clone(),
5869            daemon_observed: DaemonObservedProcess {
5870                pid: self.daemon_provenance.pid,
5871                started_at_ms: self
5872                    .daemon_provenance
5873                    .start_clock
5874                    .map(|clock| clock.started_at_ms())
5875                    .or(self.daemon_provenance.started_at_ms),
5876                running_image: self
5877                    .daemon_provenance
5878                    .probe
5879                    .observe(
5880                        self.daemon_provenance.pid,
5881                        self.daemon_provenance.executable_path.as_deref(),
5882                        self.daemon_provenance.executable_identity.clone(),
5883                        self.daemon_provenance.process_start_time,
5884                    )
5885                    .await,
5886            },
5887        };
5888        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5889        Ok(vec![control_response_body_frame(
5890            &frame,
5891            &response,
5892            "ClientControlResponse::SupervisorProvenance",
5893        )?])
5894    }
5895
5896    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5897        self.refresh_capability_requirements();
5898        let generation = self
5899            .registry
5900            .generation()
5901            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5902        let modules = self
5903            .supervisor
5904            .list()
5905            .into_iter()
5906            .map(|module| {
5907                let status = module.status_for_control("health").map_err(|err| {
5908                    RouterError::backend(
5909                        0,
5910                        frame.header.corr,
5911                        format!("failed to read supervisor health: {err}"),
5912                    )
5913                })?;
5914                let module_id = status.module_id;
5915                let capability_detail = self
5916                    .capability_evaluator
5917                    .required_problem_detail(&module_id);
5918                Ok(SupervisorHealthEntry {
5919                    module_id,
5920                    status: status.health.status,
5921                    detail: append_capability_problem_detail(
5922                        status.health.detail,
5923                        capability_detail,
5924                    ),
5925                    metrics: status.health.metrics,
5926                    consecutive_failures: status.health.consecutive_failures,
5927                    late_answer_count: status.health.late_answer_count,
5928                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5929                    last_action: status.health.last_action,
5930                    last_action_ms: status.health.last_action_ms,
5931                    last_probe_ms: status.health.last_probe_ms,
5932                })
5933            })
5934            .collect::<Result<Vec<_>, RouterError>>()?;
5935        let response = ClientControlResponse::SupervisorHealth {
5936            generation,
5937            modules,
5938        };
5939        Ok(vec![control_response_body_frame(
5940            &frame,
5941            &response,
5942            "ClientControlResponse::SupervisorHealth",
5943        )?])
5944    }
5945
5946    async fn handle_supervisor_restart(
5947        &self,
5948        frame: Frame,
5949        module_id: String,
5950        drain_timeout_ms: Option<u64>,
5951        requester: Principal,
5952    ) -> Result<Vec<Frame>, RouterError> {
5953        let operation_lock = self.supervisor.operation_lock();
5954        let _operation_guard = operation_lock.lock().await;
5955        let Some(module) = self.supervisor.get(&module_id) else {
5956            return Ok(vec![control_error_frame(
5957                &frame,
5958                "unknown_module",
5959                format!("module_id '{module_id}' is not supervised"),
5960            )?]);
5961        };
5962
5963        self.route_outages.mark_operator_action(&module_id);
5964        if let Err(err) = module
5965            .restart_requested(drain_timeout_ms, Some(requester))
5966            .await
5967        {
5968            self.route_outages
5969                .operator_action_ended_unrefused(&module_id);
5970            let (code, message) = match err {
5971                crate::supervise::SuperviseError::Disabled { .. } => {
5972                    ("module_disabled", err.to_string())
5973                }
5974                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5975                    ("swap_in_progress", err.to_string())
5976                }
5977                _ => (
5978                    "target_unavailable",
5979                    format!("failed to restart module_id '{module_id}': {err}"),
5980                ),
5981            };
5982            return Ok(vec![control_error_frame(&frame, code, message)?]);
5983        }
5984
5985        let response = ClientControlResponse::SupervisorAck {
5986            module_id,
5987            applied: true,
5988        };
5989        Ok(vec![control_response_body_frame(
5990            &frame,
5991            &response,
5992            "ClientControlResponse::SupervisorAck",
5993        )?])
5994    }
5995
5996    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5997    /// when the old process has finished draining: a caller whose own lane
5998    /// rides the old process must get its reply before that drain waits on it.
5999    async fn handle_supervisor_swap(
6000        &self,
6001        frame: Frame,
6002        module_id: String,
6003        ready_timeout_ms: Option<u64>,
6004        requester: Principal,
6005    ) -> Result<Vec<Frame>, RouterError> {
6006        // The daemon-wide operation lock is held only to resolve the handle,
6007        // not across the swap. The swap can take its whole readiness budget,
6008        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6009        // holding it here would park an operator's stop behind the swap it is
6010        // meant to abort. A rescan or stop that reaches the module during the
6011        // swap is served by the swap itself (see `supervise_swap`).
6012        let module = {
6013            let operation_lock = self.supervisor.operation_lock();
6014            let _operation_guard = operation_lock.lock().await;
6015            self.supervisor.get(&module_id)
6016        };
6017        let Some(module) = module else {
6018            return Ok(vec![control_error_frame(
6019                &frame,
6020                "unknown_module",
6021                format!("module_id '{module_id}' is not supervised"),
6022            )?]);
6023        };
6024
6025        self.route_outages.mark_operator_action(&module_id);
6026        if let Err(err) = module
6027            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6028            .await
6029        {
6030            self.route_outages
6031                .operator_action_ended_unrefused(&module_id);
6032            use crate::supervise::SuperviseError;
6033            let message = err.to_string();
6034            let error = match err {
6035                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6036                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6037                    code: "swap_refused".to_string(),
6038                    message,
6039                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6040                },
6041                SuperviseError::SwapFailed {
6042                    arm,
6043                    candidate_exit,
6044                    ..
6045                } => ErrorBody {
6046                    code: "swap_failed".to_string(),
6047                    message,
6048                    detail: Some(serde_json::json!({
6049                        "arm": arm.as_str(),
6050                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6051                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6052                    })),
6053                },
6054                _ => ErrorBody::new(
6055                    "target_unavailable",
6056                    format!("failed to swap module_id '{module_id}': {message}"),
6057                ),
6058            };
6059            return Ok(vec![control_error_body_frame(&frame, error)?]);
6060        }
6061        // A completed swap kept the incumbent serving until cutover, so it
6062        // usually opened no outage; a mark left behind would make the next,
6063        // unrelated outage read as requested.
6064        self.route_outages
6065            .operator_action_ended_unrefused(&module_id);
6066
6067        let response = ClientControlResponse::SupervisorAck {
6068            module_id,
6069            applied: true,
6070        };
6071        Ok(vec![control_response_body_frame(
6072            &frame,
6073            &response,
6074            "ClientControlResponse::SupervisorAck",
6075        )?])
6076    }
6077
6078    async fn handle_supervisor_reload(
6079        &self,
6080        frame: Frame,
6081        module_id: String,
6082        requester: Principal,
6083    ) -> Result<Vec<Frame>, RouterError> {
6084        let operation_lock = self.supervisor.operation_lock();
6085        let _operation_guard = operation_lock.lock().await;
6086        let Some(module) = self.supervisor.get(&module_id) else {
6087            return Ok(vec![control_error_frame(
6088                &frame,
6089                "unknown_module",
6090                format!("module_id '{module_id}' is not supervised"),
6091            )?]);
6092        };
6093
6094        self.route_outages.mark_operator_action(&module_id);
6095        if let Err(err) = module.reload_requested(Some(requester)).await {
6096            self.route_outages
6097                .operator_action_ended_unrefused(&module_id);
6098            let (code, message) = match err {
6099                crate::supervise::SuperviseError::Disabled { .. } => {
6100                    ("module_disabled", err.to_string())
6101                }
6102                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6103                    ("swap_in_progress", err.to_string())
6104                }
6105                _ => (
6106                    "reload_failed",
6107                    format!("failed to reload module_id '{module_id}': {err}"),
6108                ),
6109            };
6110            return Ok(vec![control_error_frame(&frame, code, message)?]);
6111        }
6112
6113        let response = ClientControlResponse::SupervisorAck {
6114            module_id,
6115            applied: true,
6116        };
6117        Ok(vec![control_response_body_frame(
6118            &frame,
6119            &response,
6120            "ClientControlResponse::SupervisorAck",
6121        )?])
6122    }
6123
6124    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6125    /// the configuration comes from and whether to apply it; every mode then
6126    /// runs the same comparison, so a preview describes the apply it previews.
6127    async fn handle_supervisor_rescan(
6128        &self,
6129        frame: Frame,
6130        mode: RescanMode,
6131        requester: Principal,
6132    ) -> Result<Vec<Frame>, RouterError> {
6133        let preview = mode.previews();
6134        let Some(context) = self.rescan.clone() else {
6135            return Ok(vec![control_error_frame(
6136                &frame,
6137                "rescan_unavailable",
6138                "the daemon was not started with a reloadable config path".to_string(),
6139            )?]);
6140        };
6141
6142        let operation_lock = self.supervisor.operation_lock();
6143        let _operation_guard = operation_lock.lock().await;
6144        // Loaded under the operation lock, so no other rescan can change the
6145        // staged configuration between a checked apply's comparison and the
6146        // apply itself.
6147        let loaded = match self.load_rescan_config(&mode, &context) {
6148            Ok(config) => config,
6149            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6150        };
6151        // `load` reports a missing file as Ok(None), which is correct at boot
6152        // (no config, nothing to supervise) and catastrophic here: rescan treats
6153        // "not in the config" as "remove it", so an absent file would read as an
6154        // empty module list and retire the entire running fleet. An editor
6155        // writing via write-new-then-rename, or a half-finished edit, is enough
6156        // to open that window. Refuse instead: a config that cannot be read
6157        // carries no instruction to remove anything.
6158        let Some((config, activity_config)) = loaded else {
6159            return Ok(vec![control_error_frame(
6160                &frame,
6161                "invalid_daemon_config",
6162                format!(
6163                    "daemon config not found at {}; refusing to rescan (an absent config would \
6164                     retire every supervised module)",
6165                    context.config_path.display()
6166                ),
6167            )?]);
6168        };
6169        let (
6170            configured_port,
6171            storage_config,
6172            admission_facts_carrier_module_id,
6173            admission_facts_targets,
6174            scope_authority_owners,
6175            privileged_operator_prompt,
6176            modules,
6177            reserved_capabilities,
6178        ) = (
6179            config.port,
6180            config.storage,
6181            config.admission_facts_carrier_module_id,
6182            config.admission_facts_targets,
6183            config.scope_authority_owners,
6184            config.privileged_operator_prompt,
6185            config.modules,
6186            config.reserved_capabilities,
6187        );
6188
6189        // Collect the sections rescan cannot apply, so the REPLY carries them.
6190        //
6191        // The warning below has always been correct and has always gone only to
6192        // the journal -- addressed to whoever reads logs, while the person who
6193        // just edited the config is looking at the CLI. Naming each section
6194        // individually rather than setting a flag: "something outside modules
6195        // changed" sends the operator back to diffing their own file, which is
6196        // the work this is meant to save.
6197        let mut restart_required = Vec::new();
6198        for section in RestartRequiredSection::ALL {
6199            let changed = match section {
6200                RestartRequiredSection::Port => configured_port != context.configured_port,
6201                RestartRequiredSection::Storage => storage_config != context.storage_config,
6202                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6203                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6204                }
6205                RestartRequiredSection::AdmissionFactsTargets => {
6206                    admission_facts_targets != context.admission_facts_targets
6207                }
6208                RestartRequiredSection::ScopeAuthorityOwners => {
6209                    scope_authority_owners != context.scope_authority_owners
6210                }
6211                RestartRequiredSection::PrivilegedOperatorPrompt => {
6212                    privileged_operator_prompt != context.privileged_operator_prompt
6213                }
6214            };
6215            if changed {
6216                restart_required.push(section.label().to_string());
6217            }
6218        }
6219        if !restart_required.is_empty() {
6220            warn!(
6221                config_path = %context.config_path.display(),
6222                sections = %restart_required.join(", "),
6223                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6224            );
6225        }
6226
6227        for configured in &modules {
6228            if let Err(err) = validate_spec(&configured.module_spec()) {
6229                return Ok(vec![control_error_frame(
6230                    &frame,
6231                    "invalid_daemon_config",
6232                    format!("supervisor rescan rejected daemon config: {err}"),
6233                )?]);
6234            }
6235        }
6236
6237        let configured_capabilities = modules
6238            .iter()
6239            .map(|module| (module.module_id.clone(), module.enabled))
6240            .collect::<Vec<_>>();
6241        let preview_capability_warnings = if preview {
6242            let (_, registrations) = self.runtime_capability_snapshot()?;
6243            let current_modules = self
6244                .supervisor
6245                .list()
6246                .into_iter()
6247                .map(|module| module.module_id().to_string())
6248                .collect::<BTreeSet<_>>();
6249            let resulting_modules = configured_capabilities.clone();
6250            let removed = current_modules
6251                .into_iter()
6252                .filter(|module_id| {
6253                    !resulting_modules
6254                        .iter()
6255                        .any(|(configured_id, _)| configured_id == module_id)
6256                })
6257                .collect::<Vec<_>>();
6258            // A candidate preview also names providers the candidate disables.
6259            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6260            let disabled = match &mode {
6261                RescanMode::Candidate(_) => resulting_modules
6262                    .iter()
6263                    .filter(|(_, enabled)| !enabled)
6264                    .map(|(module_id, _)| module_id.clone())
6265                    .collect::<Vec<_>>(),
6266                _ => Vec::new(),
6267            };
6268            self.capability_evaluator.preview_warnings(
6269                resulting_modules,
6270                &removed,
6271                &disabled,
6272                &registrations,
6273            )
6274        } else {
6275            Vec::new()
6276        };
6277        let reconciled = match &mode {
6278            RescanMode::Candidate(candidate) => self
6279                .preview_candidate(&context, modules, candidate)
6280                .map(|result| (result, Vec::new())),
6281            _ => {
6282                self.reconcile_supervised_modules(
6283                    &context.supervisor,
6284                    modules,
6285                    preview,
6286                    requester.clone(),
6287                )
6288                .await
6289            }
6290        };
6291        let (result, changed) = match reconciled {
6292            Ok(result) => result,
6293            Err(message) => {
6294                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6295            }
6296        };
6297        if !preview {
6298            if let Some(ledger) = &self.activity {
6299                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6300                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6301            }
6302            if let Err(code) = self.configure_activity(activity_config).await {
6303                warn!(
6304                    code,
6305                    "activity retention could not be applied during rescan"
6306                );
6307            }
6308            self.capability_evaluator
6309                .configure(configured_capabilities, reserved_capabilities);
6310            self.capability_evaluator.wake_deadline_loop();
6311            self.refresh_capability_requirements();
6312        }
6313        let mut result = result;
6314        result.restart_required = restart_required;
6315        result.capability_warnings = preview_capability_warnings;
6316        let response = ClientControlResponse::SupervisorRescan { result };
6317        Ok(vec![control_response_body_frame(
6318            &frame,
6319            &response,
6320            "ClientControlResponse::SupervisorRescan",
6321        )?])
6322    }
6323
6324    async fn handle_supervisor_release_reserved(
6325        &self,
6326        frame: Frame,
6327        module_id: String,
6328    ) -> Result<Vec<Frame>, RouterError> {
6329        let Some(context) = self.rescan.clone() else {
6330            return Ok(vec![control_error_frame(
6331                &frame,
6332                "release_unavailable",
6333                "reserved-id release requires a daemon started with a reloadable config path",
6334            )?]);
6335        };
6336        let operation_lock = self.supervisor.operation_lock();
6337        let _operation_guard = operation_lock.lock().await;
6338        let loaded = match crate::daemon_config::load(&context.config_path) {
6339            Ok(Some(config)) => config,
6340            Ok(None) => {
6341                return Ok(vec![control_error_frame(
6342                    &frame,
6343                    "invalid_daemon_config",
6344                    format!(
6345                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6346                        context.config_path.display()
6347                    ),
6348                )?])
6349            }
6350            Err(err) => {
6351                return Ok(vec![control_error_frame(
6352                    &frame,
6353                    "invalid_daemon_config",
6354                    format!("unable to verify reserved-id release against daemon config: {err}"),
6355                )?])
6356            }
6357        };
6358        if loaded
6359            .modules
6360            .iter()
6361            .any(|configured| configured.module_id == module_id)
6362        {
6363            return Ok(vec![control_error_frame(
6364                &frame,
6365                "reserved_module_configured",
6366                format!(
6367                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6368                ),
6369            )?]);
6370        }
6371        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6372            return Ok(vec![control_error_frame(
6373                &frame,
6374                "reserved_gate_not_retained",
6375                format!(
6376                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6377                ),
6378            )?]);
6379        }
6380
6381        let response = ClientControlResponse::SupervisorAck {
6382            module_id,
6383            applied: true,
6384        };
6385        Ok(vec![control_response_body_frame(
6386            &frame,
6387            &response,
6388            "ClientControlResponse::SupervisorAck",
6389        )?])
6390    }
6391
6392    /// Compare the configured module set with the supervised one, changing
6393    /// nothing. The staged spec each supervised module holds is what the
6394    /// configured entry is compared with.
6395    fn plan_rescan(
6396        &self,
6397        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6398    ) -> Result<RescanPlan, String> {
6399        let mut current = BTreeMap::new();
6400        for module in self.supervisor.list() {
6401            let (spec, health) = module.configuration().map_err(|err| {
6402                format!(
6403                    "failed to read configuration for module_id '{}': {err}",
6404                    module.module_id()
6405                )
6406            })?;
6407            let enabled = module
6408                .status()
6409                .map_err(|err| {
6410                    format!(
6411                        "failed to read status for module_id '{}': {err}",
6412                        module.module_id()
6413                    )
6414                })?
6415                .enabled;
6416            current.insert(
6417                module.module_id().to_string(),
6418                (module, spec, health, enabled),
6419            );
6420        }
6421        let configured = configured_modules
6422            .into_iter()
6423            .map(|module| (module.module_id.clone(), module))
6424            .collect::<BTreeMap<_, _>>();
6425
6426        let added = configured
6427            .keys()
6428            .filter(|module_id| !current.contains_key(*module_id))
6429            .cloned()
6430            .collect::<Vec<_>>();
6431        let removed = current
6432            .keys()
6433            .filter(|module_id| !configured.contains_key(*module_id))
6434            .cloned()
6435            .collect::<Vec<_>>();
6436        let mut changed_pending_reload = Vec::new();
6437        let mut configuration_changes = BTreeSet::new();
6438        let mut enabled_changes = BTreeSet::new();
6439        let mut unchanged = 0_u32;
6440
6441        for (module_id, configured_module) in &configured {
6442            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6443            else {
6444                continue;
6445            };
6446            // Compare the whole launch spec so a future launch field cannot
6447            // accidentally become a live-only policy change. Health is stored
6448            // separately and applies live without replacing the process.
6449            let launch_changed = *current_spec != configured_module.module_spec();
6450            let configuration_changed =
6451                launch_changed || *current_health != configured_module.health;
6452            let enabled_changed = *current_enabled != configured_module.enabled;
6453            if configuration_changed {
6454                configuration_changes.insert(module_id.clone());
6455            }
6456            if launch_changed {
6457                changed_pending_reload.push(module_id.clone());
6458            }
6459            if enabled_changed {
6460                enabled_changes.insert(module_id.clone());
6461            }
6462            if !configuration_changed && !enabled_changed {
6463                unchanged = unchanged.saturating_add(1);
6464            }
6465        }
6466        Ok(RescanPlan {
6467            current,
6468            configured,
6469            added,
6470            removed,
6471            changed_pending_reload,
6472            configuration_changes,
6473            enabled_changes,
6474            unchanged,
6475        })
6476    }
6477
6478    /// Reconcile the running module set against the configured one.
6479    ///
6480    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6481    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6482    /// deliberately shares this function with the executing path rather than
6483    /// computing the same diff somewhere else -- two implementations of one
6484    /// decision agree until they do not, and the whole value of a preview is that
6485    /// it describes the operation that will actually run.
6486    async fn reconcile_supervised_modules(
6487        &self,
6488        supervisor: &Supervisor,
6489        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6490        preview: bool,
6491        requester: Principal,
6492    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6493        let plan = self.plan_rescan(configured_modules)?;
6494
6495        // Everything above this point is pure computation over two snapshots.
6496        // Everything below MUTATES. The preview returns here so the boundary is a
6497        // single early return rather than a condition repeated at each mutation
6498        // site, where one missed guard would apply part of a change the caller was
6499        // told would not happen.
6500        if preview {
6501            return Ok((plan.preview_result(), Vec::new()));
6502        }
6503        let changed = plan
6504            .configuration_changes
6505            .union(&plan.enabled_changes)
6506            .cloned()
6507            .collect::<Vec<_>>();
6508        let RescanPlan {
6509            current,
6510            configured,
6511            added,
6512            removed,
6513            changed_pending_reload,
6514            configuration_changes,
6515            enabled_changes,
6516            unchanged,
6517        } = plan;
6518        // The staged generation moves before the first change rather than
6519        // after the last, so a rescan that fails partway still invalidates
6520        // every earlier `supervisor.rescan_checked` preview: part of its diff
6521        // may have run.
6522        if !(added.is_empty()
6523            && removed.is_empty()
6524            && configuration_changes.is_empty()
6525            && enabled_changes.is_empty())
6526        {
6527            if let Some(context) = &self.rescan {
6528                context
6529                    .generation
6530                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6531            }
6532        }
6533
6534        for module_id in &removed {
6535            let module = &current
6536                .get(module_id)
6537                .expect("removed module came from current supervisor state")
6538                .0;
6539            module
6540                .retire_requested(Some(requester.clone()))
6541                .await
6542                .map_err(|err| {
6543                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6544                })?;
6545            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6546            //
6547            // `handle_route_open` resolves an absent module in three steps:
6548            // registry, then supervisor status, then tombstone. Retiring first
6549            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6550            // went with the teardown above, the supervisor entry went with
6551            // `retire`, and the tombstone does not exist yet -- so a route.open
6552            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6553            // it") for a module that was deliberately removed and whose caller
6554            // should get `module_removed` (TERMINAL, carrying a removal age).
6555            //
6556            // Writing the tombstone first closes it: during the window the
6557            // supervisor entry still answers, so the caller gets
6558            // `target_unavailable` -- retryable, and TRUE, because the module
6559            // is mid-teardown. After both statements it is `module_removed`.
6560            // No instant remains where a removed module reads as one that
6561            // never existed.
6562            //
6563            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6564            // the absence of a test beside a fix invites deletion: these are
6565            // two sync statements with no await between them, so reaching the
6566            // window needs a second worker thread to land exactly between them
6567            // and there is no hook to force it. MEASURED: the 25 daemon_config
6568            // tests pass identically with the old order and the new one, so
6569            // the existing suite cannot see this and a green run is not
6570            // evidence either way. What the suite does hold is the
6571            // post-condition -- a removed module answers `module_removed` --
6572            // which this preserves.
6573            //
6574            // Found by an Athena panel reading the shipped tree against a
6575            // design note (2026-09-19), as the one concrete instance of that
6576            // note's class that survived contact with source. Direction is
6577            // benign: retryable where terminal was intended, never the reverse.
6578            self.supervisor.record_rescan_removal(module_id);
6579            self.supervisor.retire(module_id);
6580            self.route_outages.forget(module_id);
6581        }
6582
6583        for module_id in configured.keys() {
6584            let Some((module, _, _, _)) = current.get(module_id) else {
6585                continue;
6586            };
6587            let configured_module = configured
6588                .get(module_id)
6589                .expect("configured module id came from configured map");
6590            if configuration_changes.contains(module_id) {
6591                module
6592                    .update_configuration(
6593                        configured_module.module_spec(),
6594                        configured_module.health.clone(),
6595                        configured_module.drain_timeout_ms,
6596                    )
6597                    .await
6598                    .map_err(|err| {
6599                        format!(
6600                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6601                        )
6602                    })?;
6603            }
6604            if enabled_changes.contains(module_id) {
6605                // A rescan that starts or stops a module applies an operator's
6606                // edit to the config, so the resulting outage was asked for.
6607                self.route_outages.mark_operator_action(module_id);
6608                module
6609                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6610                    .await
6611                    .map_err(|err| {
6612                        self.route_outages.operator_action_ended_unrefused(module_id);
6613                        format!(
6614                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6615                            configured_module.enabled
6616                        )
6617                    })?;
6618            }
6619        }
6620
6621        for module_id in &added {
6622            let configured_module = configured
6623                .get(module_id)
6624                .expect("added module id came from configured map");
6625            supervisor
6626                .supervise_configured_requested(
6627                    configured_module.module_spec(),
6628                    configured_module.enabled,
6629                    configured_module.health.clone(),
6630                    configured_module.drain_timeout_ms,
6631                    configured_module.restart,
6632                    configured_module.register_timeout,
6633                    Some(requester.clone()),
6634                )
6635                .map_err(|err| {
6636                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6637                })?;
6638        }
6639
6640        Ok((
6641            SupervisorRescanResult {
6642                added,
6643                removed,
6644                changed_pending_reload,
6645                enabled_changes: enabled_changes.iter().cloned().collect(),
6646                unchanged,
6647                preview: false,
6648                // Filled by the caller, which is the only layer that can see the
6649                // previous config to diff against.
6650                restart_required: Vec::new(),
6651                capability_warnings: Vec::new(),
6652                module_actions: Vec::new(),
6653                earlier_pending: Vec::new(),
6654                expect: None,
6655            },
6656            changed,
6657        ))
6658    }
6659
6660    async fn handle_supervisor_set_enabled(
6661        &self,
6662        frame: Frame,
6663        module_id: String,
6664        enabled: bool,
6665        requester: Principal,
6666    ) -> Result<Vec<Frame>, RouterError> {
6667        let operation_lock = self.supervisor.operation_lock();
6668        let _operation_guard = operation_lock.lock().await;
6669        let Some(module) = self.supervisor.get(&module_id) else {
6670            return Ok(vec![control_error_frame(
6671                &frame,
6672                "unknown_module",
6673                format!("module_id '{module_id}' is not supervised"),
6674            )?]);
6675        };
6676
6677        // Enabling counts as well as disabling: a module an operator starts
6678        // is refused until it registers, and that wait was asked for.
6679        self.route_outages.mark_operator_action(&module_id);
6680        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6681            Ok(applied) => applied,
6682            Err(err) => {
6683                self.route_outages
6684                    .operator_action_ended_unrefused(&module_id);
6685                return Ok(vec![control_error_frame(
6686                    &frame,
6687                    "target_unavailable",
6688                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6689                )?]);
6690            }
6691        };
6692        if !applied {
6693            // Already in the requested state: nothing was made unavailable,
6694            // so the mark must not outlive this request.
6695            self.route_outages
6696                .operator_action_ended_unrefused(&module_id);
6697        }
6698
6699        self.capability_evaluator.wake_deadline_loop();
6700        self.refresh_capability_requirements();
6701        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6702        Ok(vec![control_response_body_frame(
6703            &frame,
6704            &response,
6705            "ClientControlResponse::SupervisorAck",
6706        )?])
6707    }
6708
6709    async fn handle_supervisor_health_probe(
6710        &self,
6711        frame: Frame,
6712        module_id: String,
6713    ) -> Result<Vec<Frame>, RouterError> {
6714        self.refresh_capability_requirements();
6715        let Some(registration) = self
6716            .registry
6717            .get_module(&module_id)
6718            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6719        else {
6720            return Ok(vec![control_error_frame(
6721                &frame,
6722                "unknown_module",
6723                format!("module_id '{module_id}' is not registered"),
6724            )?]);
6725        };
6726
6727        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6728        // named for it. Making `module_registration_grants_op` return false
6729        // unconditionally reddens five tests, and every one is named for something
6730        // else -- capability relay, probe/bind demultiplexing, supervision-only
6731        // probing. They exercise a successful advertisement check on the way to their
6732        // own subject.
6733        //
6734        // Real protection, fragile in a specific way: narrowing any of those tests to
6735        // focus on its stated subject would silently remove coverage nobody knows
6736        // they are carrying. Recorded here rather than as a sixth test, because the
6737        // useful fact is WHICH tests hold the guard up -- a new test would add
6738        // coverage without telling the next person what the existing ones quietly do.
6739        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6740        {
6741            return Ok(vec![control_error_frame(
6742                &frame,
6743                "health_not_advertised",
6744                format!("module_id '{module_id}' did not advertise health.check"),
6745            )?]);
6746        }
6747
6748        let deadline = Instant::now() + self.health_probe_timeout;
6749        let pending = match self.forwarding.begin_module_control_rpc_for(
6750            &module_id,
6751            MODULE_CONTROL_OP_HEALTH_CHECK,
6752            deadline,
6753        ) {
6754            Ok(pending) => pending,
6755            Err(err) => {
6756                return Ok(vec![control_error_frame(
6757                    &frame,
6758                    forwarding_error_code(&err),
6759                    err.to_string(),
6760                )?])
6761            }
6762        };
6763
6764        let PendingModuleControlRpc {
6765            endpoint,
6766            module_sink,
6767            negotiated_ver,
6768            corr: probe_corr,
6769            receiver,
6770        } = pending;
6771        let mut guard =
6772            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6773        let probe_body =
6774            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6775                RouterError::backend(
6776                    0,
6777                    frame.header.corr,
6778                    format!("failed to encode health.check request: {err}"),
6779                )
6780            })?;
6781        let probe_frame = Frame::build_with_version(
6782            negotiated_ver,
6783            FrameType::Request,
6784            control_flags(),
6785            0,
6786            0,
6787            probe_corr,
6788            probe_body,
6789        )
6790        .map_err(RouterError::FrameBuild)?;
6791
6792        // Queue admission and the reply share one deadline, so a module that
6793        // stops reading cannot park the requesting connection's reader forever.
6794        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6795            Ok(Ok(())) => {}
6796            Ok(Err(err)) => {
6797                return Ok(vec![control_error_frame(
6798                    &frame,
6799                    "target_unavailable",
6800                    err.to_string(),
6801                )?]);
6802            }
6803            Err(_) => {
6804                return Ok(vec![control_error_frame(
6805                    &frame,
6806                    "module_timeout",
6807                    format!(
6808                        "module_id '{module_id}' did not answer health.check within {:?}",
6809                        self.health_probe_timeout
6810                    ),
6811                )?]);
6812            }
6813        }
6814
6815        match timeout_at(deadline, receiver).await {
6816            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6817                guard.disarm();
6818                let Some(report) = response.health_report() else {
6819                    return Ok(vec![control_error_frame(
6820                        &frame,
6821                        "invalid_control_body",
6822                        "health.check RPC returned a non-health response",
6823                    )?]);
6824                };
6825                // Metrics go out whole here. The supervisor's cached snapshot
6826                // caps this blob (see truncate_health_metrics), and this path
6827                // exists precisely to answer without that cap -- so applying it
6828                // here would leave no way to see what the cached view drops.
6829                let HealthReport {
6830                    status,
6831                    detail,
6832                    metrics,
6833                } = report;
6834                let capability_detail = self
6835                    .capability_evaluator
6836                    .required_problem_detail(&module_id);
6837                let response = ClientControlResponse::SupervisorHealthProbe {
6838                    module_id,
6839                    status,
6840                    detail: append_capability_problem_detail(detail, capability_detail),
6841                    metrics,
6842                };
6843                Ok(vec![control_response_body_frame(
6844                    &frame,
6845                    &response,
6846                    "ClientControlResponse::SupervisorHealthProbe",
6847                )?])
6848            }
6849            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6850                guard.disarm();
6851                // The module refused the health probe with its own body, so
6852                // the refusal is relayed without the daemon-origin bit.
6853                Ok(vec![control_relayed_error_body_frame(&frame, body)?])
6854            }
6855            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6856                guard.disarm();
6857                Ok(vec![control_error_frame(
6858                    &frame,
6859                    "target_unavailable",
6860                    message,
6861                )?])
6862            }
6863            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6864                guard.disarm();
6865                Ok(vec![control_error_frame(
6866                    &frame,
6867                    "invalid_control_body",
6868                    message,
6869                )?])
6870            }
6871            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6872                guard.disarm();
6873                Ok(vec![control_error_frame(
6874                    &frame,
6875                    "invalid_control_body",
6876                    format!("expected module-control op '{expected}', got '{actual}'"),
6877                )?])
6878            }
6879            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6880                guard.disarm();
6881                Ok(vec![control_error_frame(
6882                    &frame,
6883                    "module_timeout",
6884                    format!(
6885                        "module_id '{module_id}' answered health.check after {:?}",
6886                        self.health_probe_timeout
6887                    ),
6888                )?])
6889            }
6890            Ok(Err(_)) => Ok(vec![control_error_frame(
6891                &frame,
6892                "target_unavailable",
6893                "health.check waiter was canceled before the module responded",
6894            )?]),
6895            Err(_) => Ok(vec![control_error_frame(
6896                &frame,
6897                "module_timeout",
6898                format!(
6899                    "module_id '{module_id}' did not answer health.check within {:?}",
6900                    self.health_probe_timeout
6901                ),
6902            )?]),
6903        }
6904    }
6905
6906    fn supervisor_status(
6907        &self,
6908        module_id: &str,
6909        corr: u64,
6910    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6911        self.supervisor
6912            .get(module_id)
6913            .map(|module| {
6914                let warming = module.is_warming_for_control("status").map_err(|err| {
6915                    RouterError::backend(
6916                        0,
6917                        corr,
6918                        format!(
6919                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6920                        ),
6921                    )
6922                })?;
6923                module.status_for_control("status").map_err(|err| {
6924                    RouterError::backend(
6925                        0,
6926                        corr,
6927                        format!(
6928                            "failed to read supervisor status for module_id '{module_id}': {err}"
6929                        ),
6930                    )
6931                }).map(|status| (status, warming))
6932            })
6933            .transpose()
6934    }
6935
6936    fn guard_module_control_op(
6937        &self,
6938        frame: &Frame,
6939        module_id: &str,
6940        op: &str,
6941    ) -> Result<Option<Frame>, RouterError> {
6942        if self.module_grants_op(module_id, op, frame.header.corr)? {
6943            return Ok(None);
6944        }
6945
6946        Ok(Some(control_error_frame(
6947            frame,
6948            "op_not_allowed",
6949            format!("module_id '{module_id}' did not grant control op '{op}'"),
6950        )?))
6951    }
6952
6953    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6954        let Some(registration) = self
6955            .registry
6956            .get_module(module_id)
6957            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6958        else {
6959            return Ok(false);
6960        };
6961        Ok(module_registration_grants_op(&registration.control_ops, op))
6962    }
6963
6964    fn handle_status_update(
6965        &self,
6966        endpoint: ModuleEndpointId,
6967        frame: Frame,
6968    ) -> Result<Vec<Frame>, RouterError> {
6969        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6970            Ok(update) => update,
6971            Err(err) => {
6972                // Forward-compat: a newer module may push a channel-0 op this subc
6973                // version doesn't know. The control contract says unknown push ops
6974                // are IGNORED, never answered with an error. Only a malformed body
6975                // for an op we DO know is a real error worth surfacing.
6976                if is_known_module_push_op(&frame.body) {
6977                    return Ok(vec![control_error_frame(
6978                        &frame,
6979                        "invalid_control_body",
6980                        format!("malformed module control push body: {err}"),
6981                    )?]);
6982                }
6983                return Ok(Vec::new());
6984            }
6985        };
6986
6987        match update {
6988            ModuleControlPush::RouteStatus {
6989                route_channel,
6990                route_epoch,
6991                status,
6992            } => {
6993                self.forwarding
6994                    .cache_status(endpoint, route_channel, route_epoch, status)
6995                    .map_err(RouterError::Forwarding)?;
6996            }
6997        }
6998        Ok(Vec::new())
6999    }
7000
7001    fn handle_route_poll(
7002        &self,
7003        ctx: &RouteCtx,
7004        frame: Frame,
7005        route_channel: u16,
7006        route_epoch: u32,
7007        kind: PollKind,
7008    ) -> Result<Vec<Frame>, RouterError> {
7009        let snapshot = self
7010            .forwarding
7011            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7012            .map_err(RouterError::Forwarding)?;
7013        let response = match (kind, snapshot) {
7014            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7015                ClientControlResponse::RoutePoll {
7016                    route_channel,
7017                    route_epoch,
7018                    status,
7019                    live: None,
7020                }
7021            }
7022            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7023                route_channel,
7024                route_epoch,
7025                status: None,
7026                live: None,
7027            },
7028            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7029                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7030                // is what makes reporting `true` correct rather than a
7031                // confident guess. `process_live` returns None only when the
7032                // module id has no supervisor snapshot at all -- an
7033                // externally-started module the daemon did not spawn -- and
7034                // for those the supervisor has no opinion to offer, ever. It
7035                // is never None for a supervised module in an unknown state:
7036                // a supervised module always has a snapshot, and the answer
7037                // comes from `state == Running && process_alive`.
7038                //
7039                // The route is Bound, so the module completed a HELLO on a
7040                // live connection; "the process this route points at is
7041                // running" is therefore attested by the binding rather than
7042                // assumed. Reporting `false` for an unsupervised module would
7043                // be the actual lie -- it would tell a client its healthy
7044                // route is dead because the daemon does not manage the
7045                // process.
7046                //
7047                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7048                // module whose liveness is genuinely unknown, e.g. a snapshot
7049                // that has not been populated yet -- THIS DEFAULT BECOMES
7050                // WRONG and must split: unsupervised stays true, unknown
7051                // becomes null so the client can tell the two apart. The
7052                // response field is already `Option<bool>`, so the wire can
7053                // carry that distinction today.
7054                let live = self
7055                    .process_liveness
7056                    .as_ref()
7057                    .and_then(|source| source.process_live(&module_id))
7058                    .unwrap_or(true);
7059                ClientControlResponse::RoutePoll {
7060                    route_channel,
7061                    route_epoch,
7062                    status: None,
7063                    live: Some(live),
7064                }
7065            }
7066            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7067                route_channel,
7068                route_epoch,
7069                status: None,
7070                live: Some(false),
7071            },
7072        };
7073
7074        Ok(vec![control_response_body_frame(
7075            &frame,
7076            &response,
7077            "ClientControlResponse::RoutePoll",
7078        )?])
7079    }
7080
7081    pub(crate) fn observe_module_control_completion(
7082        &self,
7083        completion: ModuleControlRpcCompletion,
7084    ) -> bool {
7085        match completion {
7086            ModuleControlRpcCompletion::Unknown => false,
7087            ModuleControlRpcCompletion::Settled => true,
7088            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7089                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7090                info!(
7091                    module_id = %module_id,
7092                    latency_ms,
7093                    "late health.check answer proves the module is alive"
7094                );
7095                match self
7096                    .supervisor
7097                    .record_late_health_answer(&module_id, latency_ms)
7098                {
7099                    Ok(true) => {}
7100                    Ok(false) => debug!(
7101                        module_id = %module_id,
7102                        latency_ms,
7103                        "late health.check answer has no active supervisor snapshot"
7104                    ),
7105                    Err(err) => warn!(
7106                        module_id = %module_id,
7107                        latency_ms,
7108                        error = %err,
7109                        "failed to record late health.check answer"
7110                    ),
7111                }
7112                true
7113            }
7114        }
7115    }
7116
7117    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7118    /// the module connection whose frame is being handled, or to the client that
7119    /// relay was opened for.
7120    ///
7121    /// This runs on the MODULE connection's frame handler, where returning `Err`
7122    /// ends that connection -- and a module connection carries every client's
7123    /// routes to that module, so ending it costs the whole fleet its tools.
7124    /// `ConnectionClosing` carries the id of the connection that is closing, and
7125    /// when that id is a CLIENT's, the condition is entirely about that one
7126    /// client's route.open. A client-scoped condition has no authority over a
7127    /// shared module connection, so it is logged and the single relay is dropped:
7128    /// the client is going away, and `complete_pending_relay` already removed the
7129    /// relay before failing, so there is nothing left to settle. Anything that
7130    /// relay still reserved is released by that client's own connection teardown,
7131    /// which is already under way -- that is what "closing" means.
7132    ///
7133    /// Every other failure is a statement about THIS connection and stays fatal:
7134    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7135    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7136    /// frames correctly.
7137    fn refuse_to_end_module_connection_for_a_client(
7138        &self,
7139        module_connection_id: ConnectionId,
7140        corr: u64,
7141        err: ForwardingError,
7142    ) -> Result<(), RouterError> {
7143        if let ForwardingError::ConnectionClosing { connection_id } = err {
7144            if connection_id != module_connection_id {
7145                warn!(
7146                    module_connection_id = module_connection_id.get(),
7147                    client_connection_id = connection_id.get(),
7148                    corr,
7149                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7150                );
7151                return Ok(());
7152            }
7153        }
7154        Err(RouterError::Forwarding(err))
7155    }
7156
7157    fn handle_module_relay_response(
7158        &self,
7159        connection_id: ConnectionId,
7160        frame: Frame,
7161    ) -> Result<Vec<Frame>, RouterError> {
7162        let mut secondary_error = None;
7163        let outcome = match frame.header.ty {
7164            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7165                Ok(probe) if probe.op == "route.bind" => {
7166                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7167                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7168                            RouteBindRelayOutcome::Accepted
7169                        }
7170                        Ok(other) => {
7171                            let message =
7172                                format!("route.bind response carried unexpected body: {other:?}");
7173                            secondary_error = Some(control_error_frame(
7174                                &frame,
7175                                "invalid_control_body",
7176                                message.clone(),
7177                            )?);
7178                            RouteBindRelayOutcome::ModuleGone(message)
7179                        }
7180                        Err(err) => {
7181                            let message = format!("malformed route.bind response body: {err}");
7182                            secondary_error = Some(control_error_frame(
7183                                &frame,
7184                                "invalid_control_body",
7185                                message.clone(),
7186                            )?);
7187                            RouteBindRelayOutcome::ModuleGone(message)
7188                        }
7189                    }
7190                }
7191                Ok(probe) => {
7192                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7193                    {
7194                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7195                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7196                            "malformed {} response body: {err}",
7197                            probe.op
7198                        )),
7199                    };
7200                    let completion = self
7201                        .forwarding
7202                        .complete_module_control_rpc(
7203                            connection_id,
7204                            frame.header.corr,
7205                            Some(&probe.op),
7206                            outcome,
7207                        )
7208                        .map_err(RouterError::Forwarding)?;
7209                    if !self.observe_module_control_completion(completion) {
7210                        debug!(
7211                            connection_id = connection_id.get(),
7212                            corr = frame.header.corr,
7213                            op = %probe.op,
7214                            "dropping late or unknown module-control RPC response"
7215                        );
7216                    }
7217                    return Ok(Vec::new());
7218                }
7219                Err(err) => {
7220                    if let Some(expected_op) = self
7221                        .forwarding
7222                        .pending_module_control_op(connection_id, frame.header.corr)
7223                        .map_err(RouterError::Forwarding)?
7224                    {
7225                        let completion = self
7226                            .forwarding
7227                            .complete_module_control_rpc(
7228                                connection_id,
7229                                frame.header.corr,
7230                                None,
7231                                ModuleControlRpcOutcome::MalformedResponse(format!(
7232                                    "malformed {expected_op} response body: {err}"
7233                                )),
7234                            )
7235                            .map_err(RouterError::Forwarding)?;
7236                        if !self.observe_module_control_completion(completion) {
7237                            debug!(
7238                                connection_id = connection_id.get(),
7239                                corr = frame.header.corr,
7240                                "dropping late malformed module-control RPC response"
7241                            );
7242                        }
7243                        return Ok(Vec::new());
7244                    }
7245                    let message = format!("malformed route.bind response body: {err}");
7246                    secondary_error = Some(control_error_frame(
7247                        &frame,
7248                        "invalid_control_body",
7249                        message.clone(),
7250                    )?);
7251                    RouteBindRelayOutcome::ModuleGone(message)
7252                }
7253            },
7254            FrameType::Error => {
7255                if self
7256                    .forwarding
7257                    .pending_module_control_op(connection_id, frame.header.corr)
7258                    .map_err(RouterError::Forwarding)?
7259                    .is_some()
7260                {
7261                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7262                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7263                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7264                            "malformed module-control ERROR body: {err}"
7265                        )),
7266                    };
7267                    let completion = self
7268                        .forwarding
7269                        .complete_module_control_rpc(
7270                            connection_id,
7271                            frame.header.corr,
7272                            None,
7273                            outcome,
7274                        )
7275                        .map_err(RouterError::Forwarding)?;
7276                    if !self.observe_module_control_completion(completion) {
7277                        debug!(
7278                            connection_id = connection_id.get(),
7279                            corr = frame.header.corr,
7280                            "dropping late or unknown module-control RPC error"
7281                        );
7282                    }
7283                    return Ok(Vec::new());
7284                }
7285                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7286                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7287                    Err(err) => {
7288                        let message = format!("malformed route.bind ERROR body: {err}");
7289                        secondary_error = Some(control_error_frame(
7290                            &frame,
7291                            "invalid_control_body",
7292                            message.clone(),
7293                        )?);
7294                        RouteBindRelayOutcome::ModuleGone(message)
7295                    }
7296                }
7297            }
7298            ty => {
7299                return Ok(vec![control_error_frame(
7300                    &frame,
7301                    "unsupported_control_frame",
7302                    format!("unsupported module channel-0 frame {ty:?}"),
7303                )?])
7304            }
7305        };
7306
7307        let settled =
7308            self.forwarding
7309                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7310        let completion = match settled {
7311            Ok(completion) => completion,
7312            Err(err) => {
7313                self.refuse_to_end_module_connection_for_a_client(
7314                    connection_id,
7315                    frame.header.corr,
7316                    err,
7317                )?;
7318                return Ok(secondary_error.into_iter().collect());
7319            }
7320        };
7321        if let Some(target) = completion.abandoned.as_ref() {
7322            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7323        }
7324        if !completion.settled {
7325            debug!(
7326                connection_id = connection_id.get(),
7327                corr = frame.header.corr,
7328                frame_type = ?frame.header.ty,
7329                "dropping late or unknown route.bind relay response"
7330            );
7331        }
7332        Ok(secondary_error.into_iter().collect())
7333    }
7334
7335    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7336        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7337        // GOODBYE ends the connection's logical session even when its socket
7338        // stays open. Use disconnect teardown so verdicts, client notices and
7339        // scope authority are released at the same lifecycle boundary.
7340        self.cleanup_connection_with_end_reason(
7341            connection_id,
7342            RegistrationEndReason::ExplicitGoodbye,
7343        )
7344        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7345        Ok(Vec::new())
7346    }
7347}
7348
7349impl Default for ControlHandler {
7350    fn default() -> Self {
7351        Self::new(Arc::new(Registry::default()))
7352    }
7353}
7354
7355impl crate::supervise::SwapPromotionObserver for ControlHandler {
7356    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7357        self.apply_registration_capabilities(registration);
7358    }
7359}
7360
7361fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7362    CapabilityRequirementStatus {
7363        consumer: status.consumer,
7364        capability: status.capability,
7365        need: match status.need {
7366            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7367            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7368        },
7369        verdict: status.verdict.as_str().to_string(),
7370        episode_seq: status.episode_seq,
7371        config_satisfiable: status.config_satisfiable,
7372        runtime_available: status.runtime_available,
7373        detail: status.detail,
7374    }
7375}
7376
7377fn append_capability_problem_detail(
7378    detail: Option<String>,
7379    capability_detail: Option<String>,
7380) -> Option<String> {
7381    match (detail, capability_detail) {
7382        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7383        (Some(detail), None) => Some(detail),
7384        (None, Some(capability_detail)) => Some(capability_detail),
7385        (None, None) => None,
7386    }
7387}
7388
7389fn subc_ops() -> Vec<String> {
7390    SUBC_CONTROL_OPS
7391        .iter()
7392        .map(|op| (*op).to_string())
7393        .collect()
7394}
7395
7396fn module_subc_ops() -> Vec<String> {
7397    SUBC_CONTROL_OPS
7398        .iter()
7399        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7400        .map(|op| (*op).to_string())
7401        .collect()
7402}
7403
7404#[cfg(test)]
7405fn module_baseline_control_ops() -> Vec<String> {
7406    MODULE_BASELINE_CONTROL_OPS
7407        .iter()
7408        .map(|op| (*op).to_string())
7409        .collect()
7410}
7411
7412fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7413    let mut seen = HashSet::new();
7414    let mut effective = Vec::new();
7415    for op in MODULE_BASELINE_CONTROL_OPS {
7416        if seen.insert((*op).to_string()) {
7417            effective.push((*op).to_string());
7418        }
7419    }
7420    for op in declared.unwrap_or_default() {
7421        if seen.insert(op.clone()) {
7422            effective.push(op);
7423        }
7424    }
7425    effective
7426}
7427
7428fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7429    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7430}
7431
7432fn target_module_id(target: &RouteTarget) -> &str {
7433    match target {
7434        RouteTarget::ToolProvider { module_id }
7435        | RouteTarget::ManagementSurface { module_id }
7436        | RouteTarget::InternalService { module_id, .. } => module_id,
7437    }
7438}
7439
7440fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7441    roles.iter().any(|role| match (target, role) {
7442        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7443        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7444        (
7445            RouteTarget::InternalService { service_id, .. },
7446            ProviderRole::InternalService {
7447                service_id: provided,
7448                ..
7449            },
7450        ) => service_id == provided,
7451        _ => false,
7452    })
7453}
7454
7455fn is_routable_role(role: &ProviderRole) -> bool {
7456    matches!(
7457        role,
7458        ProviderRole::ToolProvider { .. }
7459            | ProviderRole::ManagementSurface { .. }
7460            | ProviderRole::InternalService { .. }
7461    )
7462}
7463
7464#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7465enum ControlRequestBodyError {
7466    UnknownOp,
7467    InvalidBody,
7468}
7469
7470#[derive(Debug, Deserialize)]
7471struct ControlOpProbe {
7472    op: String,
7473}
7474
7475/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7476/// this set is treated as a forward-compat unknown and ignored rather than errored.
7477const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7478
7479fn is_known_module_push_op(body: &[u8]) -> bool {
7480    serde_json::from_slice::<ControlOpProbe>(body)
7481        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7482        .unwrap_or(false)
7483}
7484
7485fn is_known_module_request_op(body: &[u8]) -> bool {
7486    serde_json::from_slice::<ControlOpProbe>(body)
7487        .map(|probe| is_module_to_subc_op(&probe.op))
7488        .unwrap_or(false)
7489}
7490
7491fn is_module_to_subc_op(op: &str) -> bool {
7492    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7493}
7494
7495fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7496    debug!(
7497        op = %op,
7498        connection_id = connection_id.get(),
7499        corr,
7500        "control dispatch"
7501    );
7502}
7503
7504fn log_slow_control_dispatch(
7505    dispatch_started_at: Option<StdInstant>,
7506    op: &'static str,
7507    connection_id: ConnectionId,
7508    corr: u64,
7509    route_open_timings: Option<&RouteOpenTimings>,
7510) {
7511    let Some(dispatch_started_at) = dispatch_started_at else {
7512        return;
7513    };
7514    // Use the same monotonic clock as the phase timers and relay deadline.
7515    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7516    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7517        if let Some(timings) = route_open_timings {
7518            warn!(
7519                op = %op,
7520                connection_id = connection_id.get(),
7521                corr,
7522                elapsed_ms = elapsed.as_millis() as u64,
7523                module_id = ?timings.module_id,
7524                principal = %timings.principal,
7525                admission_ms = timings.admission.as_millis() as u64,
7526                project_root_ms = timings.project_root.as_millis() as u64,
7527                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7528                bind_send_ms = timings.bind_send.as_millis() as u64,
7529                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7530                "slow control dispatch"
7531            );
7532            return;
7533        }
7534        warn!(
7535            op = %op,
7536            connection_id = connection_id.get(),
7537            corr,
7538            elapsed_ms = elapsed.as_millis() as u64,
7539            "slow control dispatch"
7540        );
7541    }
7542}
7543
7544/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7545#[cfg(test)]
7546type TestCallerLookup =
7547    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7548
7549/// Every new client operation needs an explicit privilege decision here.
7550fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7551    match request {
7552        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7553            Some(PrivilegedOperation::Restart { module_id })
7554        }
7555        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7556            Some(PrivilegedOperation::Swap { module_id })
7557        }
7558        ClientControlRequest::SupervisorReload { module_id } => {
7559            Some(PrivilegedOperation::Reload { module_id })
7560        }
7561        ClientControlRequest::SupervisorRescan { preview } => {
7562            if *preview {
7563                None
7564            } else {
7565                Some(PrivilegedOperation::Rescan)
7566            }
7567        }
7568        // Only the well-formed apply asks for approval. A preview changes
7569        // nothing, and any other combination is refused before it touches
7570        // anything, so prompting the operator for it would ask about a request
7571        // that cannot run.
7572        ClientControlRequest::SupervisorRescanChecked {
7573            preview,
7574            candidate,
7575            expect,
7576        } => (!*preview && candidate.is_none() && expect.is_some())
7577            .then_some(PrivilegedOperation::Rescan),
7578        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7579            Some(PrivilegedOperation::SetEnabled {
7580                module_id,
7581                enabled: *enabled,
7582            })
7583        }
7584        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7585            Some(PrivilegedOperation::ReleaseReserved { module_id })
7586        }
7587        ClientControlRequest::ServerDescribe {}
7588        | ClientControlRequest::CatalogList { .. }
7589        | ClientControlRequest::RouteOpen { .. }
7590        | ClientControlRequest::RoutePoll { .. }
7591        | ClientControlRequest::SupervisorList {}
7592        | ClientControlRequest::SupervisorHealth {}
7593        | ClientControlRequest::SupervisorHealthProbe { .. }
7594        | ClientControlRequest::SupervisorRoutes { .. }
7595        | ClientControlRequest::SupervisorProvenance { .. }
7596        | ClientControlRequest::SupervisorStderrTail { .. }
7597        | ClientControlRequest::SupervisorTerminals { .. }
7598        | ClientControlRequest::SupervisorSpawnSnapshot {}
7599        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7600    }
7601}
7602
7603fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7604    match request {
7605        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7606        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7607        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7608        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7609        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7610        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7611        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7612        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7613        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7614        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7615        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7616        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7617        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7618        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7619        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7620        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7621        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7622        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7623        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7624        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7625    }
7626}
7627
7628/// The caller of one `route.open`, as its `route.open refused` line names it.
7629///
7630/// A connection id alone cannot name a caller after the fact: ids restart at
7631/// every daemon restart, so the same number belongs to different modules
7632/// before and after one. These three fields are what the refused line needs
7633/// to be attributable on its own, and they are rendered with the same helpers
7634/// as the `route.open accepted` line so both directions grep alike.
7635struct RouteOpenCaller {
7636    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7637    /// when the request presented a consumer identity whose launch nonce did
7638    /// not verify. The claimed module id is never printed as a principal.
7639    principal: String,
7640    /// `none` for an unscoped open; the admitted stamp's label once scope
7641    /// admission issued one; before that, the requested selector's label.
7642    scope: String,
7643    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7644    /// because it is a string the caller chose.
7645    harness: String,
7646}
7647
7648impl RouteOpenCaller {
7649    fn new(
7650        principal: Option<&Principal>,
7651        scope: Option<&ScopeSelector>,
7652        identity: &BindIdentity,
7653    ) -> Self {
7654        Self {
7655            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7656            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7657            harness: bounded_log_text(&identity.harness),
7658        }
7659    }
7660
7661    /// For a refusal that has no parsed request at hand: names whatever the
7662    /// frame still says, and `unknown` where it says nothing parseable.
7663    fn unknown() -> Self {
7664        Self {
7665            principal: "unknown".to_string(),
7666            scope: "unknown".to_string(),
7667            harness: "unknown".to_string(),
7668        }
7669    }
7670}
7671
7672/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7673/// prints before cutting it.
7674const MAX_LOGGED_TEXT_CHARS: usize = 64;
7675
7676/// `value` Debug-quoted, so control characters land escaped and cannot forge
7677/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7678/// `…` inside the quotes so one request cannot flood the line.
7679fn bounded_log_text(value: &str) -> String {
7680    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7681    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7682        bounded.push('…');
7683    }
7684    format!("{bounded:?}")
7685}
7686
7687/// The `scope=` field of the `route.open accepted` line, from the stamp that
7688/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7689/// so a ref holding control characters cannot forge a log line, and cut to 64
7690/// characters with a trailing `…` to bound the line. The bind proof and the
7691/// scope's attributes never appear.
7692fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7693    use subc_protocol::scope::ScopeKind;
7694    let kind = match stamp.kind {
7695        ScopeKind::Head => "head",
7696        ScopeKind::Worker => "worker",
7697        ScopeKind::Ephemeral => "ephemeral",
7698    };
7699    format!(
7700        "{kind}/{}/{}@{}",
7701        principal_label(&stamp.owner),
7702        bounded_log_text(&stamp.scope_ref),
7703        stamp.scope_epoch
7704    )
7705}
7706
7707/// The `scope=` field of a `route.open refused` line for an open refused
7708/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7709/// The kind is not known yet, so the literal `requested` stands in its place.
7710/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7711/// owner it has not been checked against the configured modules: a quoted
7712/// owner means the value was never validated. The bind proof never appears.
7713fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7714    let epoch = selector
7715        .scope_epoch
7716        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7717    format!(
7718        "requested/{}/{}@{epoch}",
7719        bounded_log_text(&principal_label(&selector.owner)),
7720        bounded_log_text(&selector.scope_ref),
7721    )
7722}
7723
7724fn principal_label(principal: &Principal) -> String {
7725    match principal {
7726        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7727        Principal::Direct => "direct".to_string(),
7728        other => format!("{other:?}"),
7729    }
7730}
7731
7732fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7733    match request {
7734        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7735        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7736        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7737        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7738        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7739    }
7740}
7741
7742fn parse_client_control_request(
7743    body: &[u8],
7744) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7745    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7746        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7747            Ok(probe)
7748                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7749                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7750            {
7751                ControlRequestBodyError::InvalidBody
7752            }
7753            Ok(_) => ControlRequestBodyError::UnknownOp,
7754            Err(_) => ControlRequestBodyError::InvalidBody,
7755        };
7756        (err, classification)
7757    })
7758}
7759
7760fn parse_module_control_request_from_module(
7761    body: &[u8],
7762) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7763    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7764        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7765            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7766            Ok(_) => ControlRequestBodyError::UnknownOp,
7767            Err(_) => ControlRequestBodyError::InvalidBody,
7768        };
7769        (err, classification)
7770    })
7771}
7772
7773#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7774enum ProviderRoleKind {
7775    ToolProvider,
7776    PipelineStage,
7777    ManagementSurface,
7778    InternalService,
7779}
7780
7781fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7782    match role {
7783        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7784        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7785        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7786        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7787    }
7788}
7789
7790fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7791    roles.iter().map(provider_role_kind).collect()
7792}
7793
7794/// Most refused scope records named individually in the log per sync; the
7795/// `refused` count on the accepted line is always complete.
7796const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7797
7798/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7799#[derive(Debug, Default, PartialEq, Eq)]
7800struct ScopeOutcomeCounts {
7801    created: usize,
7802    replaced: usize,
7803    updated: usize,
7804    unchanged: usize,
7805    refused: usize,
7806}
7807
7808impl ScopeOutcomeCounts {
7809    fn of(results: &[ScopeRecordResult]) -> Self {
7810        let mut counts = Self::default();
7811        for result in results {
7812            let slot = match result.outcome {
7813                ScopeRecordOutcome::Created => &mut counts.created,
7814                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7815                ScopeRecordOutcome::Updated => &mut counts.updated,
7816                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7817                ScopeRecordOutcome::Refused => &mut counts.refused,
7818            };
7819            *slot += 1;
7820        }
7821        counts
7822    }
7823}
7824
7825#[cfg(test)]
7826mod scope_outcome_count_tests {
7827    use super::*;
7828
7829    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7830        ScopeRecordResult {
7831            scope_ref: "r".to_string(),
7832            scope_epoch: 1,
7833            outcome,
7834            code: None,
7835            message: None,
7836            version: None,
7837            parent_state: None,
7838        }
7839    }
7840
7841    /// Each outcome lands in its own count, so a refused record can never be
7842    /// hidden inside the total the log already printed.
7843    #[test]
7844    fn every_outcome_is_counted_in_its_own_field() {
7845        let results = [
7846            result(ScopeRecordOutcome::Created),
7847            result(ScopeRecordOutcome::Created),
7848            result(ScopeRecordOutcome::Replaced),
7849            result(ScopeRecordOutcome::Updated),
7850            result(ScopeRecordOutcome::Unchanged),
7851            result(ScopeRecordOutcome::Refused),
7852            result(ScopeRecordOutcome::Refused),
7853            result(ScopeRecordOutcome::Refused),
7854        ];
7855        assert_eq!(
7856            ScopeOutcomeCounts::of(&results),
7857            ScopeOutcomeCounts {
7858                created: 2,
7859                replaced: 1,
7860                updated: 1,
7861                unchanged: 1,
7862                refused: 3,
7863            }
7864        );
7865    }
7866}
7867
7868/// Return whether a catalog change can create a newly violating live route.
7869/// Removing an attested claim is intentionally excluded: it makes fewer routes
7870/// forbidden and therefore must leave the existing route census untouched.
7871fn capability_census_trigger(
7872    old: Option<&CapabilityDeclarations>,
7873    new: Option<&CapabilityDeclarations>,
7874) -> bool {
7875    let old_provides = old
7876        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7877        .unwrap_or_default();
7878    let old_denies = old
7879        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7880        .unwrap_or_default();
7881    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7882        provides: Vec::new(),
7883        requires: Vec::new(),
7884        must_never_reach: Vec::new(),
7885    });
7886
7887    new.provides
7888        .iter()
7889        .any(|capability| !old_provides.contains(capability))
7890        || new
7891            .must_never_reach
7892            .iter()
7893            .any(|capability| !old_denies.contains(capability))
7894}
7895
7896/// Find the first capability an attested opener denies that an attested target
7897/// claims. Both manifests are live registry records, never cached or client data.
7898fn denied_capability<'a>(
7899    opening_manifest: &'a ModuleManifest,
7900    target_manifest: &ModuleManifest,
7901) -> Option<&'a str> {
7902    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7903    let target_capabilities = target_manifest.capabilities.as_ref()?;
7904    opening_capabilities
7905        .must_never_reach
7906        .iter()
7907        .find(|denied| {
7908            target_capabilities
7909                .provides
7910                .iter()
7911                .any(|provided| provided == *denied)
7912        })
7913        .map(String::as_str)
7914}
7915
7916fn catalog_update_frozen_field_message(
7917    registered: &ModuleManifest,
7918    provides: &[ProviderRole],
7919) -> Option<String> {
7920    let old_has_provides = !registered.provides.is_empty();
7921    let new_has_provides = !provides.is_empty();
7922    if old_has_provides != new_has_provides {
7923        return Some(format!(
7924            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7925            registered.module_id
7926        ));
7927    }
7928
7929    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7930        return Some(format!(
7931            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7932            registered.module_id
7933        ));
7934    }
7935
7936    let registered_concurrency = manifest_concurrency(registered);
7937    let mut candidate = registered.clone();
7938    candidate.provides = provides.to_vec();
7939    let candidate_concurrency = manifest_concurrency(&candidate);
7940    if candidate_concurrency != registered_concurrency {
7941        return Some(format!(
7942            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7943            registered.module_id, registered_concurrency, candidate_concurrency
7944        ));
7945    }
7946
7947    // control_ops live beside the manifest in the HELLO body, not inside
7948    // ModuleManifest, so a provides-only catalog.update cannot change them.
7949    None
7950}
7951
7952fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7953    manifest.provides.iter().any(is_routable_role)
7954}
7955
7956/// Returns the routable-provider concurrency subc should enforce for this manifest.
7957///
7958/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7959/// InternalService has no role-specific concurrency field, so it retains the
7960/// existing ModuleManaged default for backward compatibility.
7961fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7962    manifest
7963        .provides
7964        .iter()
7965        .find_map(|provider| match provider {
7966            ProviderRole::ToolProvider { concurrency, .. }
7967            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7968            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7969        })
7970        .unwrap_or(Concurrency::ModuleManaged)
7971}
7972
7973/// True when the manifest carries a ManagementSurface role whose concurrency
7974/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7975/// bytes because the typed manifest deliberately erases that distinction: the
7976/// default exists for wire compatibility, and this probe exists so the default
7977/// stays observable. Any parse irregularity returns false -- the caller only
7978/// logs, and a malformed body already failed registration upstream.
7979fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7980    let has_management_surface = manifest
7981        .provides
7982        .iter()
7983        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7984    if !has_management_surface {
7985        return false;
7986    }
7987    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7988        return false;
7989    };
7990    let Some(provides) = raw
7991        .get("manifest")
7992        .and_then(|manifest| manifest.get("provides"))
7993        .and_then(serde_json::Value::as_array)
7994    else {
7995        return false;
7996    };
7997    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7998    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7999    // against the management_surface_manifest_without_concurrency golden, not
8000    // recalled (the externally-tagged guess was this function's first bug).
8001    provides.iter().any(|role| {
8002        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8003            && role.get("concurrency").is_none()
8004    })
8005}
8006
8007fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8008    if peer_version != PROTOCOL_VERSION {
8009        return Err(format!(
8010            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8011        ));
8012    }
8013    Ok(PROTOCOL_VERSION)
8014}
8015
8016fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8017    Frame::build_with_version(
8018        response_version(frame),
8019        FrameType::Pong,
8020        frame.header.flags,
8021        0,
8022        0,
8023        frame.header.corr,
8024        Vec::new(),
8025    )
8026    .map_err(RouterError::FrameBuild)
8027}
8028
8029pub(crate) fn control_error_frame(
8030    frame: &Frame,
8031    code: &'static str,
8032    message: impl Into<String>,
8033) -> Result<Frame, RouterError> {
8034    control_error_body_frame(
8035        frame,
8036        ErrorBody {
8037            code: code.to_string(),
8038            message: message.into(),
8039            detail: None,
8040        },
8041    )
8042}
8043
8044/// A channel-0 ERROR for a refusal the daemon decided itself: every
8045/// route.open refusal, the drain `module_reloading` and scope-change/end
8046/// bodies, spawn-subscription and supervisor refusals. Carries the
8047/// daemon-origin bit, which is what lets a client treat `module_reloading`
8048/// as proof that its request never reached a module. Response, push,
8049/// goodbye and relay-request frames keep the plain control flags in their
8050/// own builders.
8051fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8052    control_error_frame_with_flags(frame, error, daemon_error_flags())
8053}
8054
8055/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8056/// refused route.bind answered as a route.open refusal, or a refused
8057/// module-control RPC. Plain control flags — the code and message are
8058/// module-chosen, so the frame must not claim daemon authorship, even when
8059/// the module picks a daemon code like `module_reloading`.
8060fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8061    control_error_frame_with_flags(frame, error, control_flags())
8062}
8063
8064fn control_error_frame_with_flags(
8065    frame: &Frame,
8066    error: ErrorBody,
8067    flags: Flags,
8068) -> Result<Frame, RouterError> {
8069    let body = serde_json::to_vec(&error).map_err(|err| {
8070        RouterError::backend(
8071            0,
8072            frame.header.corr,
8073            format!("failed to encode control ERROR: {err}"),
8074        )
8075    })?;
8076
8077    Frame::build_with_version(
8078        response_version(frame),
8079        FrameType::Error,
8080        flags,
8081        0,
8082        0,
8083        frame.header.corr,
8084        body,
8085    )
8086    .map_err(RouterError::FrameBuild)
8087}
8088
8089fn control_response_body_frame<T: Serialize>(
8090    frame: &Frame,
8091    reply: &T,
8092    label: &'static str,
8093) -> Result<Frame, RouterError> {
8094    let body = serde_json::to_vec(reply).map_err(|err| {
8095        RouterError::backend(
8096            0,
8097            frame.header.corr,
8098            format!("failed to encode {label}: {err}"),
8099        )
8100    })?;
8101
8102    Frame::build_with_version(
8103        response_version(frame),
8104        FrameType::Response,
8105        control_flags(),
8106        0,
8107        0,
8108        frame.header.corr,
8109        body,
8110    )
8111    .map_err(RouterError::FrameBuild)
8112}
8113
8114/// Map a forwarding failure to the wire code a client sees.
8115///
8116/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8117/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8118/// chosen here decides whether a caller retries or gives up.
8119///
8120/// That makes attribution the load-bearing property, not merely having a code. A
8121/// permanent fault published as a retryable one produces a fleet-wide retry storm
8122/// against something that can never recover; a transient fault published as
8123/// permanent gives up on work that would have succeeded. Both look correct in a
8124/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8125/// the mapping per variant rather than merely asserting that some code exists.
8126///
8127/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8128/// two codes on the same side of the boundary passes it. Measured rather than
8129/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8130/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8131/// a test named for something else that happens to assert the string.
8132///
8133/// That accidental coverage is deliberately left alone rather than promoted to a
8134/// named test, because it guards a property this function does not promise.
8135/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8136/// specific code within a class, so identity is free to change and only the
8137/// partition is a contract. Splitting it out would assert a guarantee nothing
8138/// depends on — and a suite that promises more than the code does is the harder
8139/// thing to correct later, because the next reader cannot tell which assertions
8140/// are load-bearing.
8141///
8142/// Pin identity here the moment a consumer branches on a specific code.
8143fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8144    match err {
8145        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8146        ForwardingError::NoModuleConnection => "target_unavailable",
8147        ForwardingError::ModuleReloading { .. } => "module_reloading",
8148        ForwardingError::ClientRouteChannelExhausted { .. }
8149        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8150        ForwardingError::StaleModuleEndpoint
8151        | ForwardingError::UnknownReservation { .. }
8152        | ForwardingError::ConnectionClosing { .. }
8153        | ForwardingError::ClientEgressClosed { .. }
8154        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8155        // Only a swap candidate's registration can produce this, and it means
8156        // exactly what a second active HELLO for a live id means.
8157        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8158        ForwardingError::RelayCorrelationExhausted
8159        | ForwardingError::RouteOpenBuild(_)
8160        | ForwardingError::Poisoned => "forwarding_error",
8161    }
8162}
8163
8164fn response_version(frame: &Frame) -> u8 {
8165    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8166        frame.header.ver
8167    } else {
8168        PROTOCOL_VERSION
8169    }
8170}
8171
8172fn control_flags() -> Flags {
8173    Flags::new(false, Priority::Passive, false)
8174}
8175
8176/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8177/// channel. The target is the module (a client never saw the route), so this
8178/// takes the module path: delivered late rather than dropped when the module's
8179/// queue is momentarily full, and never closing its connection.
8180fn send_goodbye_target_best_effort(
8181    counters: &DaemonCounters,
8182    target: &GoodbyeTarget,
8183    context: &'static str,
8184) {
8185    let Ok(frame) = Frame::build_with_version(
8186        target.negotiated_ver,
8187        FrameType::Goodbye,
8188        control_flags(),
8189        target.channel,
8190        target.epoch,
8191        0,
8192        Vec::new(),
8193    ) else {
8194        return;
8195    };
8196    crate::forwarding::send_module_route_goodbye(
8197        counters,
8198        &target.sink,
8199        frame,
8200        target.module_id.as_deref(),
8201        context,
8202    );
8203}
8204
8205pub(crate) fn send_route_control_pushes(
8206    forwarding: &ForwardingTable,
8207    routes: Vec<EndpointRoute>,
8208    push: ClientControlPush,
8209) {
8210    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8211    for route in routes {
8212        let target = route.goodbye_target;
8213        if let Some((existing, channels)) = targets
8214            .iter_mut()
8215            .find(|(existing, _)| existing.connection_id == target.connection_id)
8216        {
8217            debug_assert_eq!(
8218                existing.negotiated_ver, target.negotiated_ver,
8219                "one connection cannot negotiate multiple frame versions"
8220            );
8221            if !channels.contains(&target.channel) {
8222                channels.push(target.channel);
8223            }
8224            continue;
8225        }
8226        let channel = target.channel;
8227        targets.push((target, vec![channel]));
8228    }
8229    for (target, mut channels) in targets {
8230        channels.sort_unstable();
8231        let mut push = push.clone();
8232        match &mut push {
8233            ClientControlPush::RouteClosing {
8234                channels: covered, ..
8235            }
8236            | ClientControlPush::RouteClosed {
8237                channels: covered, ..
8238            } => *covered = channels,
8239        }
8240        let body = match serde_json::to_vec(&push) {
8241            Ok(body) => body,
8242            Err(err) => {
8243                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8244                continue;
8245            }
8246        };
8247        let frame = match Frame::build_with_version(
8248            target.negotiated_ver,
8249            FrameType::Push,
8250            control_flags(),
8251            0,
8252            0,
8253            0,
8254            body.clone(),
8255        ) {
8256            Ok(frame) => frame,
8257            Err(err) => {
8258                warn!(
8259                    route_channel = target.channel,
8260                    error = %err,
8261                    "failed to build route lifecycle control PUSH frame"
8262                );
8263                continue;
8264            }
8265        };
8266        if let Err(err) = target.sink.try_send(frame) {
8267            if target.close_on_delivery_failure() {
8268                warn!(
8269                    target_connection_id = target.connection_id.get(),
8270                    route_channel = target.channel,
8271                    error = %err,
8272                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8273                );
8274                let _ = forwarding.escalate_client_delivery_failure(
8275                    target.connection_id,
8276                    target.channel,
8277                    target.epoch,
8278                    CloseReason::new(
8279                        "route_lifecycle_push_delivery_failed",
8280                        format!(
8281                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8282                            target.channel
8283                        ),
8284                    ),
8285                    crate::forwarding::UndeliveredFrame {
8286                        module_id: target.module_id.as_deref(),
8287                        sink: &target.sink,
8288                    },
8289                );
8290            }
8291        }
8292    }
8293}
8294
8295#[cfg(test)]
8296mod tests {
8297    mod peer_manifest {
8298        include!("peer_manifest_tests.rs");
8299    }
8300    mod activity_read {
8301        include!("activity_read_tests.rs");
8302    }
8303
8304    mod lifecycle_activity {
8305        include!("control_lifecycle_tests.rs");
8306    }
8307    #[cfg(unix)]
8308    #[tokio::test]
8309    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8310        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8311        let path = dir.join("subc.jsonc");
8312        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8313            "program":"/bin/sleep","args":["60"],"protocol":"none",
8314            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8315        }}}).to_string()).unwrap();
8316        let mut configured = crate::daemon_config::load(&path)
8317            .unwrap()
8318            .unwrap()
8319            .modules
8320            .pop()
8321            .unwrap();
8322        let registry = std::sync::Arc::new(crate::Registry::default());
8323        let handle = crate::SupervisorHandle::new();
8324        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8325            .with_handle(handle.clone());
8326        let module = supervisor
8327            .supervise_configured_with_health(
8328                configured.module_spec(),
8329                true,
8330                configured.health.clone(),
8331                None,
8332                configured.restart,
8333            )
8334            .unwrap();
8335        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8336        let before = module.status().unwrap().pid;
8337        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8338        configured.health.cadence = std::time::Duration::from_secs(3600);
8339        let health_only = handler
8340            .reconcile_supervised_modules(
8341                &supervisor,
8342                vec![configured.clone()],
8343                false,
8344                Principal::Direct,
8345            )
8346            .await
8347            .unwrap();
8348        assert!(
8349            health_only.0.changed_pending_reload.is_empty(),
8350            "health policy is already applied live"
8351        );
8352        assert_eq!(module.status().unwrap().pid, before);
8353        assert_eq!(
8354            module.configuration().unwrap().1.http,
8355            configured.health.http
8356        );
8357        configured.args = vec!["61".into()];
8358        let launch = handler
8359            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8360            .await
8361            .unwrap();
8362        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8363        assert_eq!(
8364            module.status().unwrap().pid,
8365            before,
8366            "a launch edit is stored until reload"
8367        );
8368        module.drain().await.unwrap();
8369    }
8370    use cortexkit_test_support::ScratchDir;
8371    use std::{
8372        collections::BTreeMap,
8373        fmt,
8374        path::PathBuf,
8375        sync::{Arc, Mutex},
8376        time::Duration,
8377    };
8378
8379    use serde_json::{json, Value};
8380    use subc_protocol::{
8381        manifest::{
8382            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8383            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8384        },
8385        session::HealthStatus,
8386        FrameType,
8387    };
8388
8389    use super::*;
8390    use crate::{
8391        forwarding::{DataRoute, DataRouteState},
8392        registry::ChannelState,
8393        router::FrameSink,
8394        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8395        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8396        RouteCtx, Router,
8397    };
8398    use tokio::{
8399        sync::mpsc,
8400        time::{sleep, Instant},
8401    };
8402    use tracing::{
8403        field::{Field, Visit},
8404        Event, Subscriber,
8405    };
8406    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8407
8408    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8409    ///
8410    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8411    /// is only populated for `tests/*.rs` integration test binaries -- this file
8412    /// compiles as part of the library target, which gets neither. This test's
8413    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8414    /// and the sibling binary lives two directories up at
8415    /// `<target-dir>/<profile>/fake-aft-stub`.
8416    ///
8417    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8418    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8419    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8420    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8421    /// `NotFound`, which reads as a broken test rather than an unbuilt
8422    /// dependency -- so state the cause and the remedy instead. Deliberately a
8423    /// panic and not a silent skip: a test that quietly passes when it could not
8424    /// run is worse than one that fails, because it reports health it never
8425    /// verified.
8426    fn fake_aft_stub_path() -> PathBuf {
8427        let mut path = std::env::current_exe().expect("current_exe available in tests");
8428        path.pop(); // .../deps/
8429        path.pop(); // .../<profile>/
8430        path.push(if cfg!(windows) {
8431            "fake-aft-stub.exe"
8432        } else {
8433            "fake-aft-stub"
8434        });
8435        assert!(
8436            path.exists(),
8437            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8438             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8439            path.display()
8440        );
8441        path
8442    }
8443
8444    /// Whether clients retry `code` in place: the predicate itself, never a copy
8445    /// of its set. A copied list breaks silently when a code is added to or
8446    /// removed from the real one, and a stale copy here would let exactly the
8447    /// failure this test exists to catch pass.
8448    fn client_retries(code: &str) -> bool {
8449        subc_protocol::error_codes::is_retryable_route_open(code)
8450    }
8451
8452    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8453    /// of failure is worse than publishing none. A permanent fault dressed as
8454    /// retryable makes every client in the fleet retry forever against something
8455    /// that cannot recover; a transient fault dressed as permanent abandons work
8456    /// that would have succeeded.
8457    ///
8458    /// Asserting "a code exists" cannot catch either, because the string is free
8459    /// to say anything. This enumerates every variant and pins which side of the
8460    /// retry boundary it lands on, so a new variant must be classified here
8461    /// deliberately rather than inheriting whichever arm it was appended to.
8462    #[test]
8463    fn retryability_of_forwarding_codes_matches_the_failure() {
8464        // Transient by nature: the target is booting, reloading, or its endpoint
8465        // was swapped mid-flight. Retrying is how these resolve.
8466        let transient = [
8467            ForwardingError::NoModuleConnection,
8468            ForwardingError::ModuleReloading {
8469                module_id: "m".into(),
8470            },
8471            ForwardingError::StaleModuleEndpoint,
8472            ForwardingError::UnknownReservation {
8473                client_channel: 1,
8474                module_channel: 1,
8475            },
8476            ForwardingError::ConnectionClosing {
8477                connection_id: ConnectionId::new(1),
8478            },
8479            ForwardingError::ClientEgressClosed {
8480                connection_id: ConnectionId::new(1),
8481            },
8482            ForwardingError::ModuleEgressUnavailable {
8483                connection_id: ConnectionId::new(1),
8484            },
8485        ];
8486        for err in transient {
8487            let code = forwarding_error_code(&err);
8488            assert!(
8489                client_retries(code),
8490                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8491            );
8492        }
8493
8494        // Not fixed by retrying. Channel and correlation exhaustion need the
8495        // caller to close routes, and a poisoned lock is a daemon that cannot
8496        // recover at all — the worst thing to advertise as retryable, since every
8497        // client would storm a daemon that will never answer.
8498        let permanent = [
8499            ForwardingError::ConnectionRoleConflict {
8500                connection_id: ConnectionId::new(1),
8501            },
8502            ForwardingError::ClientRouteChannelExhausted {
8503                connection_id: ConnectionId::new(1),
8504            },
8505            ForwardingError::ModuleRouteChannelExhausted {
8506                endpoint: ModuleEndpointId {
8507                    connection_id: ConnectionId::new(1),
8508                    generation: 1,
8509                },
8510            },
8511            ForwardingError::RelayCorrelationExhausted,
8512            ForwardingError::RouteOpenBuild("x".into()),
8513            ForwardingError::Poisoned,
8514        ];
8515        for err in permanent {
8516            let code = forwarding_error_code(&err);
8517            assert!(
8518                !client_retries(code),
8519                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8520            );
8521        }
8522    }
8523
8524    /// The principal is the daemon's answer to "who is calling", and modules
8525    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8526    /// plexus gates connector invocation. So a stamp is an authorization input in
8527    /// another process, not a label — and both possible answers SUCCEED, which is
8528    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8529    /// first-party capability to something that never proved it; a supervised one
8530    /// stamped `Direct` silently strips a module of capability it is entitled to.
8531    ///
8532    /// Neither shows up in a test that only checks the bind succeeded. Before this
8533    /// test the only coverage was accidental —
8534    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8535    /// stamped principal on its way past, so narrowing that wire-shape test to its
8536    /// stated subject would have deleted the last assertion on this value. It
8537    /// still asserts the stamp, which is now redundancy rather than the only
8538    /// guard: both fail under the same mutation, and this one names the reason.
8539    /// SCOPE: this handler's supervisor has spawned nothing, so
8540    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8541    /// is unreachable here. Both assertions below are refusals, and a mutant that
8542    /// refuses everything would satisfy them.
8543    ///
8544    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8545    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8546    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8547    /// at source rather than assumed, since a citation is a claim about another
8548    /// file and ages like one. Recorded because a harness that structurally
8549    /// cannot reach an arm reports "none" for that arm identically to one that
8550    /// covers it and found nothing.
8551    #[tokio::test]
8552    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8553        let handler = ControlHandler::default();
8554        let frame =
8555            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8556
8557        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8558        // any process holding the connection file. Nothing was proved, so nothing
8559        // may be granted beyond the unattested floor.
8560        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8561        assert_eq!(
8562            stamped,
8563            Principal::Direct,
8564            "a caller that proved nothing must not be stamped as a supervised module"
8565        );
8566
8567        // A claimed module_id with a nonce no supervised child was given is a
8568        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8569        // quietly demoted to Direct, or an impersonation attempt looks identical
8570        // to an ordinary unattested connection.
8571        let forged = handler
8572            .route_open_principal(
8573                &frame,
8574                Some(ConsumerIdentity {
8575                    module_id: "aft".to_string(),
8576                    launch_nonce: "not-a-real-nonce".to_string(),
8577                }),
8578            )
8579            .unwrap();
8580        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8581        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8582    }
8583
8584    /// The test above hands `route_open_principal` an identity it built itself,
8585    /// which proves the stamping rule and nothing about where the identity comes
8586    /// from. The real producer is a wire body, and the two are joined by a serde
8587    /// field name that nothing else asserts.
8588    ///
8589    /// That join fails quietly in one specific way: an unrecognised key is simply
8590    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8591    /// yields `None` and every supervised module silently drops to `Direct`.
8592    /// Capability-wise that is the safe direction, but it surfaces far from its
8593    /// cause — as a module mysteriously refused bash — and it would pass every
8594    /// test that builds its own input.
8595    ///
8596    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8597    /// would break every client the moment the daemon gains a field, trading a
8598    /// quiet demotion for a hard refusal on additive change. Asserting the join
8599    /// instead means a rename breaks a test here rather than the fleet.
8600    #[test]
8601    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8602        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8603        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8604        let ClientControlRequest::RouteOpen {
8605            consumer_identity, ..
8606        } = parsed
8607        else {
8608            panic!("route.open body must parse as RouteOpen");
8609        };
8610        assert_eq!(
8611            consumer_identity,
8612            Some(ConsumerIdentity {
8613                module_id: "aft".to_string(),
8614                launch_nonce: "n".to_string(),
8615            }),
8616            "the wire field name must reach the value route_open_principal reads"
8617        );
8618    }
8619
8620    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8621        ModuleManifest::builder(module_id, "0.1.0")
8622            .protocol_ver(protocol_ver)
8623            .provides(vec![ProviderRole::ToolProvider {
8624                tools: vec![Tool {
8625                    name: "read".to_string(),
8626                    description: None,
8627                    execution_mode: ExecutionMode::Pure,
8628                    schema: json!({"type": "object"}),
8629                }],
8630                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8631                concurrency: Concurrency::ModuleManaged,
8632                emits_push: true,
8633                sub_supervises: true,
8634            }])
8635            .build()
8636    }
8637
8638    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8639        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8640    }
8641
8642    fn hello_frame_with_control_ops(
8643        module_id: &str,
8644        protocol_ver: u8,
8645        corr: u64,
8646        control_ops: Option<Vec<String>>,
8647    ) -> Frame {
8648        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8649    }
8650
8651    fn hello_frame_with_nonce(
8652        module_id: &str,
8653        protocol_ver: u8,
8654        corr: u64,
8655        launch_nonce: Option<&str>,
8656    ) -> Frame {
8657        hello_frame_full(
8658            module_id,
8659            protocol_ver,
8660            corr,
8661            None,
8662            launch_nonce.map(ToOwned::to_owned),
8663        )
8664    }
8665
8666    fn hello_frame_full(
8667        module_id: &str,
8668        protocol_ver: u8,
8669        corr: u64,
8670        control_ops: Option<Vec<String>>,
8671        launch_nonce: Option<String>,
8672    ) -> Frame {
8673        let body = serde_json::to_vec(&ModuleHelloBody {
8674            manifest: manifest(module_id, protocol_ver),
8675            protocol_ver,
8676            control_ops,
8677            launch_nonce,
8678        })
8679        .unwrap();
8680        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8681    }
8682
8683    fn hello_frame_with_rich_ui(
8684        module_id: &str,
8685        protocol_ver: u8,
8686        corr: u64,
8687        launch_nonce: Option<&str>,
8688        rich_ui: Value,
8689    ) -> Frame {
8690        let mut body = serde_json::to_value(ModuleHelloBody {
8691            manifest: manifest(module_id, protocol_ver),
8692            protocol_ver,
8693            control_ops: None,
8694            launch_nonce: launch_nonce.map(str::to_string),
8695        })
8696        .unwrap();
8697        body["manifest"]["rich_ui"] = rich_ui;
8698        Frame::build(
8699            FrameType::Hello,
8700            control_flags(),
8701            0,
8702            0,
8703            corr,
8704            serde_json::to_vec(&body).unwrap(),
8705        )
8706        .unwrap()
8707    }
8708
8709    fn non_routable_hello_frame_with_control_ops(
8710        module_id: &str,
8711        corr: u64,
8712        control_ops: Option<Vec<String>>,
8713    ) -> Frame {
8714        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8715        manifest.provides.clear();
8716        let body = serde_json::to_vec(&ModuleHelloBody {
8717            manifest,
8718            protocol_ver: PROTOCOL_VERSION,
8719            control_ops,
8720            launch_nonce: None,
8721        })
8722        .unwrap();
8723        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8724    }
8725
8726    fn capability_grammar_hello_frame(
8727        capabilities: Value,
8728        runtime_computed: Option<Value>,
8729        corr: u64,
8730    ) -> Frame {
8731        let mut body = serde_json::to_value(ModuleHelloBody {
8732            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8733            protocol_ver: PROTOCOL_VERSION,
8734            control_ops: None,
8735            launch_nonce: None,
8736        })
8737        .expect("HELLO body serializes");
8738        body["manifest"]["capabilities"] = capabilities;
8739        if let Some(runtime_computed) = runtime_computed {
8740            body["runtime_computed"] = runtime_computed;
8741        }
8742        Frame::build(
8743            FrameType::Hello,
8744            control_flags(),
8745            0,
8746            0,
8747            corr,
8748            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8749        )
8750        .expect("HELLO frame builds")
8751    }
8752
8753    fn channel_request(channel: u16, corr: u64) -> Frame {
8754        Frame::build(
8755            FrameType::Request,
8756            Flags::new(true, Priority::Interactive, false),
8757            channel,
8758            0,
8759            corr,
8760            b"opaque".to_vec(),
8761        )
8762        .unwrap()
8763    }
8764
8765    fn route_ctx(
8766        connection_id: ConnectionId,
8767    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8768        let (tx, rx) = mpsc::channel(8);
8769        (
8770            RouteCtx {
8771                connection_id,
8772                egress: FrameSink::new(tx),
8773            },
8774            rx,
8775        )
8776    }
8777
8778    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8779        serde_json::from_slice(&frame.body).unwrap()
8780    }
8781
8782    /// Register a module over a connection that has a sink and return the
8783    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8784    /// module's own sink rather than returning it as a reply, so the ack is
8785    /// taken off `rx` here and whatever the test reads next is what followed it.
8786    async fn hello_via_sink(
8787        handler: &ControlHandler,
8788        ctx: &RouteCtx,
8789        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8790        hello: Frame,
8791    ) -> Frame {
8792        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8793        assert!(
8794            replies.is_empty(),
8795            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8796        );
8797        let ack = rx
8798            .try_recv()
8799            .expect("HELLO_ACK is queued on the module sink")
8800            .frame;
8801        assert_eq!(ack.header.ty, FrameType::HelloAck);
8802        ack
8803    }
8804
8805    fn parse_error(frame: &Frame) -> Value {
8806        serde_json::from_slice(&frame.body).unwrap()
8807    }
8808
8809    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8810        serde_json::from_slice(&frame.body).unwrap()
8811    }
8812
8813    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8814        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8815            route_channel,
8816            route_epoch: 0,
8817            kind,
8818        })
8819        .unwrap();
8820        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8821    }
8822
8823    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8824        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8825            module_id: module_id.to_string(),
8826        })
8827        .unwrap();
8828        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8829    }
8830
8831    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8832        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8833    }
8834
8835    fn route_open_frame_with_consumer_capabilities(
8836        corr: u64,
8837        module_id: &str,
8838        project_root: ScratchDir,
8839        consumer_capabilities: Option<Vec<String>>,
8840    ) -> Frame {
8841        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8842            target: RouteTarget::ToolProvider {
8843                module_id: module_id.to_string(),
8844            },
8845            identity: BindIdentity::new(
8846                project_root.path().to_path_buf(),
8847                "unit".to_string(),
8848                "session".to_string(),
8849            ),
8850            consumer_identity: None,
8851            consumer_capabilities,
8852            role_versions: None,
8853            admission_facts: None,
8854            scope: None,
8855        })
8856        .unwrap();
8857        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8858    }
8859
8860    fn route_open_frame_with_role_versions(
8861        corr: u64,
8862        module_id: &str,
8863        project_root: ScratchDir,
8864        role_versions: Option<BTreeMap<String, String>>,
8865    ) -> Frame {
8866        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8867            target: RouteTarget::ToolProvider {
8868                module_id: module_id.to_string(),
8869            },
8870            identity: BindIdentity::new(
8871                project_root.path().to_path_buf(),
8872                "unit".to_string(),
8873                format!("session-{corr}"),
8874            ),
8875            consumer_identity: None,
8876            consumer_capabilities: None,
8877            role_versions,
8878            admission_facts: None,
8879            scope: None,
8880        })
8881        .unwrap();
8882        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8883    }
8884
8885    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8886        entries
8887            .iter()
8888            .map(|(role, version)| (role.to_string(), version.to_string()))
8889            .collect()
8890    }
8891
8892    fn route_open_frame_with_admission_facts(
8893        corr: u64,
8894        module_id: &str,
8895        project_root: ScratchDir,
8896        consumer_identity: Option<subc_control::ConsumerIdentity>,
8897        facts: Option<Value>,
8898    ) -> Frame {
8899        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8900            target: RouteTarget::ToolProvider {
8901                module_id: module_id.to_string(),
8902            },
8903            identity: BindIdentity::new(
8904                project_root.path().to_path_buf(),
8905                "unit".to_string(),
8906                format!("session-{corr}"),
8907            ),
8908            consumer_identity,
8909            consumer_capabilities: None,
8910            role_versions: None,
8911            admission_facts: facts,
8912            scope: None,
8913        })
8914        .unwrap();
8915        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8916    }
8917
8918    #[derive(Clone, Default)]
8919    struct EventCapture {
8920        events: Arc<Mutex<Vec<CapturedEvent>>>,
8921    }
8922
8923    #[derive(Clone, Debug)]
8924    struct CapturedEvent {
8925        target: String,
8926        level: tracing::Level,
8927        fields: BTreeMap<String, String>,
8928    }
8929
8930    impl EventCapture {
8931        fn events(&self) -> Vec<CapturedEvent> {
8932            self.events.lock().unwrap().clone()
8933        }
8934    }
8935
8936    impl<S> Layer<S> for EventCapture
8937    where
8938        S: Subscriber,
8939    {
8940        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8941            let mut visitor = EventFieldVisitor::default();
8942            event.record(&mut visitor);
8943            self.events.lock().unwrap().push(CapturedEvent {
8944                target: event.metadata().target().to_string(),
8945                level: *event.metadata().level(),
8946                fields: visitor.fields,
8947            });
8948        }
8949    }
8950
8951    #[derive(Default)]
8952    struct EventFieldVisitor {
8953        fields: BTreeMap<String, String>,
8954    }
8955
8956    impl Visit for EventFieldVisitor {
8957        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8958            self.fields
8959                .insert(field.name().to_string(), format!("{value:?}"));
8960        }
8961    }
8962
8963    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8964        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8965            status,
8966            detail: Some("warming".to_string()),
8967            metrics: Some(json!({"queue_depth": 3})),
8968        })
8969        .unwrap();
8970        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8971    }
8972
8973    fn route_bind_ack(corr: u64) -> Frame {
8974        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8975        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8976    }
8977
8978    fn unique_project_root(label: &str) -> ScratchDir {
8979        ScratchDir::new(label)
8980    }
8981
8982    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8983        match parse_route_poll(frame) {
8984            ClientControlResponse::RoutePoll {
8985                status: None,
8986                live: Some(live),
8987                ..
8988            } => assert_eq!(live, expected_live),
8989            other => panic!("unexpected route.poll response: {other:?}"),
8990        }
8991    }
8992
8993    fn bind_liveness_route(
8994        registry: &Registry,
8995        forwarding: &ForwardingTable,
8996        module_id: &str,
8997    ) -> (RouteCtx, u16, u32) {
8998        let module_connection = ConnectionId::new(101);
8999        let client_connection = ConnectionId::new(202);
9000        let registration = registry
9001            .register_with_control_ops(
9002                manifest(module_id, PROTOCOL_VERSION),
9003                PROTOCOL_VERSION,
9004                module_connection,
9005                module_baseline_control_ops(),
9006            )
9007            .unwrap();
9008        let (module_tx, _module_rx) = mpsc::channel(8);
9009        let endpoint = forwarding
9010            .register_module_connection(
9011                module_connection,
9012                module_id.to_string(),
9013                PROTOCOL_VERSION,
9014                manifest_concurrency(&registration.manifest),
9015                FrameSink::new(module_tx),
9016            )
9017            .unwrap();
9018        let (client_ctx, _client_rx) = route_ctx(client_connection);
9019        let pending = forwarding
9020            .begin_route_bind_relay_for_test(
9021                client_connection,
9022                client_ctx.egress.clone(),
9023                1,
9024                module_id,
9025            )
9026            .unwrap();
9027        assert_eq!(pending.endpoint, endpoint);
9028        let route_channel = pending.client_channel;
9029        let route_epoch = pending.client_epoch;
9030        forwarding
9031            .complete_pending_relay(
9032                module_connection,
9033                pending.corr,
9034                RouteBindRelayOutcome::Accepted,
9035            )
9036            .unwrap();
9037        (client_ctx, route_channel, route_epoch)
9038    }
9039
9040    struct FakeProcessLiveness {
9041        live: Option<bool>,
9042    }
9043
9044    impl ModuleProcessLiveness for FakeProcessLiveness {
9045        fn process_live(&self, _module_id: &str) -> Option<bool> {
9046            self.live
9047        }
9048    }
9049
9050    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9051    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9052    {
9053        let registry = Arc::new(Registry::default());
9054        let supervisor_handle = SupervisorHandle::new();
9055        let supervisor = Supervisor::new_for_test(
9056            Arc::clone(&registry),
9057            RestartPolicy::new(1, Duration::from_millis(10)),
9058        )
9059        .with_handle(supervisor_handle.clone());
9060        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9061        let module = supervisor
9062            .spawn(ModuleSpec {
9063                module_id: "stderr-tail-wire".to_string(),
9064                program: fake_aft_stub_path(),
9065                args: Vec::new(),
9066                env: vec![
9067                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9068                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9069                ],
9070                reserved: false,
9071                reserved_prefixes: Vec::new(),
9072                protocol: ModuleProtocol::Subc,
9073                overlap: Default::default(),
9074            })
9075            .unwrap();
9076
9077        let deadline = Instant::now() + Duration::from_secs(5);
9078        loop {
9079            let tail = module.stderr_tail(None, None);
9080            if tail
9081                .entries
9082                .iter()
9083                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9084                && tail.entries.iter().any(|entry| {
9085                    matches!(
9086                        entry,
9087                        TailEntry::Line {
9088                            truncated: true,
9089                            ..
9090                        }
9091                    )
9092                })
9093            {
9094                break;
9095            }
9096            assert!(
9097                Instant::now() < deadline,
9098                "module did not produce a truncated line and restart boundary: {tail:?}"
9099            );
9100            sleep(Duration::from_millis(10)).await;
9101        }
9102
9103        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9104        let request = ClientControlRequest::SupervisorStderrTail {
9105            module_id: "stderr-tail-wire".to_string(),
9106            max_lines: None,
9107            max_bytes: None,
9108        };
9109        let frame = Frame::build(
9110            FrameType::Request,
9111            control_flags(),
9112            0,
9113            0,
9114            1,
9115            serde_json::to_vec(&request).unwrap(),
9116        )
9117        .unwrap();
9118        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9119        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9120        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9121            serde_json::from_slice(&responses[0].body).unwrap()
9122        else {
9123            panic!("expected supervisor.stderr_tail response");
9124        };
9125
9126        assert!(
9127            tail.entries
9128                .iter()
9129                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9130            "the control response lost the restart boundary"
9131        );
9132        let Some(StderrTailEntry::Line {
9133            text,
9134            truncated,
9135            at_ms,
9136        }) = tail.entries.iter().find(|entry| {
9137            matches!(
9138                entry,
9139                StderrTailEntry::Line {
9140                    truncated: true,
9141                    ..
9142                }
9143            )
9144        })
9145        else {
9146            panic!("the control response lost the truncated line");
9147        };
9148        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9149        assert!(*truncated);
9150        assert!(
9151            at_ms.is_some(),
9152            "the control response lost the line's capture time"
9153        );
9154    }
9155
9156    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9157    /// read done on the worker thread would stall every other task until it
9158    /// finished; the read must run off the worker so this test's own task keeps
9159    /// running while the read is paused.
9160    #[tokio::test(flavor = "current_thread")]
9161    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9162        let dir = ScratchDir::new("terminals-off-worker");
9163        let journal_path = dir.join("terminals.jsonl");
9164        let registry = Arc::new(Registry::default());
9165        let supervisor_handle = SupervisorHandle::new();
9166        let supervisor =
9167            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9168                .with_handle(supervisor_handle.clone())
9169                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9170        let module = supervisor
9171            .spawn(ModuleSpec {
9172                module_id: "terminal-off-worker".to_string(),
9173                program: fake_aft_stub_path(),
9174                args: Vec::new(),
9175                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9176                reserved: false,
9177                reserved_prefixes: Vec::new(),
9178                protocol: ModuleProtocol::Subc,
9179                overlap: Default::default(),
9180            })
9181            .unwrap();
9182        let deadline = Instant::now() + Duration::from_secs(5);
9183        while module.terminal_history().entries.len() != 2 {
9184            assert!(Instant::now() < deadline, "module did not record two exits");
9185            sleep(Duration::from_millis(10)).await;
9186        }
9187
9188        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9189        let handler =
9190            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9191        let frame = Frame::build(
9192            FrameType::Request,
9193            control_flags(),
9194            0,
9195            0,
9196            1,
9197            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9198                module_id: "terminal-off-worker".to_string(),
9199            })
9200            .unwrap(),
9201        )
9202        .unwrap();
9203        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9204        let spawned_at = std::time::Instant::now();
9205        let read = tokio::spawn({
9206            let handler = Arc::clone(&handler);
9207            async move { handler.handle_control_frame(&ctx, frame).await }
9208        });
9209        // Waiting for the pause from a blocking thread keeps this task pending,
9210        // so the runtime's single worker is free to run the read task.
9211        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9212            .await
9213            .unwrap()
9214            .expect("the history read reached its pause");
9215        let elapsed = spawned_at.elapsed();
9216        assert!(
9217            elapsed < Duration::from_secs(2) && !read.is_finished(),
9218            "this task could not run while the history read was paused \
9219             (resumed after {elapsed:?}, read finished: {})",
9220            read.is_finished()
9221        );
9222
9223        drop(release);
9224        let responses = read.await.unwrap().unwrap();
9225        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9226        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9227            panic!("expected supervisor.terminals response");
9228        };
9229        assert_eq!(terminals.entries.len(), 2);
9230        assert_eq!(terminals.journal_skipped_lines, 0);
9231        assert_eq!(terminals.journal_read_errors, 0);
9232    }
9233
9234    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9235    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9236        let registry = Arc::new(Registry::default());
9237        let supervisor_handle = SupervisorHandle::new();
9238        let supervisor =
9239            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9240                .with_handle(supervisor_handle.clone());
9241        let module = supervisor
9242            .spawn(ModuleSpec {
9243                module_id: "terminal-golden".to_string(),
9244                program: fake_aft_stub_path(),
9245                args: Vec::new(),
9246                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9247                reserved: false,
9248                reserved_prefixes: Vec::new(),
9249                protocol: ModuleProtocol::Subc,
9250                overlap: Default::default(),
9251            })
9252            .unwrap();
9253
9254        let deadline = Instant::now() + Duration::from_secs(5);
9255        while module.terminal_history().entries.len() != 2 {
9256            assert!(
9257                Instant::now() < deadline,
9258                "module did not retain two terminal exits: {:?}",
9259                module.terminal_history()
9260            );
9261            sleep(Duration::from_millis(10)).await;
9262        }
9263
9264        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9265        let request = ClientControlRequest::SupervisorTerminals {
9266            module_id: "terminal-golden".to_string(),
9267        };
9268        let frame = Frame::build(
9269            FrameType::Request,
9270            control_flags(),
9271            0,
9272            0,
9273            1,
9274            serde_json::to_vec(&request).unwrap(),
9275        )
9276        .unwrap();
9277        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9278        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9279        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9280        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9281            panic!("expected supervisor.terminals response");
9282        };
9283        assert_eq!(terminals.entries.len(), 2);
9284        assert_eq!(terminals.dropped, 0);
9285
9286        let mut rendered = serde_json::to_value(response).unwrap();
9287        // Wall-clock fields are the observation contract, but not stable fixture
9288        // bytes; normalize only them after the real handler has shaped the response.
9289        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9290        for (index, entry) in rendered["entries"]
9291            .as_array_mut()
9292            .expect("terminal response entries array")
9293            .iter_mut()
9294            .enumerate()
9295        {
9296            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9297        }
9298
9299        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9300            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9301        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9302        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9303            std::fs::write(&golden_path, &serialized).unwrap();
9304        }
9305        let expected: Value =
9306            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9307        assert_eq!(rendered, expected);
9308    }
9309
9310    #[test]
9311    fn hello_registers_manifest_and_returns_ack() {
9312        let registry = Arc::new(Registry::default());
9313        let handler = ControlHandler::new(Arc::clone(&registry));
9314        let conn = ConnectionId::new(1);
9315
9316        let responses = handler
9317            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9318            .unwrap();
9319
9320        assert_eq!(responses.len(), 1);
9321        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9322        assert_eq!(responses[0].header.channel, 0);
9323        assert_eq!(responses[0].header.corr, 7);
9324        let ack = parse_ack(&responses[0]);
9325        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9326        assert!(ack
9327            .subc_capabilities
9328            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9329        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9330        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9331        assert!(ack
9332            .subc_ops
9333            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9334        assert!(ack
9335            .subc_ops
9336            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9337
9338        let registration = registry.get_module("aft").unwrap().unwrap();
9339        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9340        assert_eq!(registration.state, ChannelState::Active);
9341        assert_eq!(registration.connection_id, conn);
9342        assert_eq!(registration.control_ops, module_baseline_control_ops());
9343    }
9344
9345    #[test]
9346    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9347        let invalid_identifiers = [
9348            ("case_change", "credentials-Provider/v1"),
9349            ("leading_zero", "credentials-provider/v01"),
9350            ("trailing_hyphen", "credentials-provider-/v1"),
9351            ("consecutive_hyphens", "credentials--provider/v1"),
9352            ("uppercase", "Credentials-provider/v1"),
9353            ("missing_v", "credentials-provider/1"),
9354            ("whitespace", "credentials provider/v1"),
9355            ("zero_version", "credentials-provider/v0"),
9356            ("out_of_range_version", "credentials-provider/v4294967296"),
9357            (
9358                "overlength_name",
9359                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9360            ),
9361        ];
9362        let mut cases = invalid_identifiers
9363            .into_iter()
9364            .map(|(name, identifier)| {
9365                (
9366                    format!("identifier_{name}"),
9367                    "capabilities.provides[0]".to_string(),
9368                    identifier.to_string(),
9369                    json!({ "provides": [identifier] }),
9370                    None,
9371                )
9372            })
9373            .collect::<Vec<_>>();
9374        cases.extend([
9375            (
9376                "unknown_need".to_string(),
9377                "capabilities.requires[0].need".to_string(),
9378                "deferred".to_string(),
9379                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9380                None,
9381            ),
9382            (
9383                "duplicate_provides".to_string(),
9384                "capabilities.provides[1]".to_string(),
9385                "credentials-provider/v1".to_string(),
9386                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9387                None,
9388            ),
9389            (
9390                "duplicate_must_never_reach".to_string(),
9391                "capabilities.must_never_reach[1]".to_string(),
9392                "credentials-provider/v1".to_string(),
9393                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9394                None,
9395            ),
9396            (
9397                "duplicate_requires_same_need".to_string(),
9398                "capabilities.requires[1]".to_string(),
9399                "credentials-provider/v1".to_string(),
9400                json!({ "requires": [
9401                    { "capability": "credentials-provider/v1", "need": "required" },
9402                    { "capability": "credentials-provider/v1", "need": "required" }
9403                ] }),
9404                None,
9405            ),
9406            (
9407                "duplicate_requires_conflicting_need".to_string(),
9408                "capabilities.requires[1]".to_string(),
9409                "credentials-provider/v1".to_string(),
9410                json!({ "requires": [
9411                    { "capability": "credentials-provider/v1", "need": "required" },
9412                    { "capability": "credentials-provider/v1", "need": "optional" }
9413                ] }),
9414                None,
9415            ),
9416            (
9417                "capabilities_root_pointer".to_string(),
9418                "runtime_computed[0]".to_string(),
9419                "/capabilities".to_string(),
9420                json!({}),
9421                Some(json!(["/capabilities"])),
9422            ),
9423            (
9424                "capabilities_descendant_pointer".to_string(),
9425                "runtime_computed[0]".to_string(),
9426                "/capabilities/provides".to_string(),
9427                json!({}),
9428                Some(json!(["/capabilities/provides"])),
9429            ),
9430            (
9431                "malformed_pointer_without_leading_slash".to_string(),
9432                "runtime_computed[0]".to_string(),
9433                "capabilities".to_string(),
9434                json!({}),
9435                Some(json!(["capabilities"])),
9436            ),
9437            (
9438                "malformed_pointer_escape".to_string(),
9439                "runtime_computed[0]".to_string(),
9440                "/roles/~2/tools".to_string(),
9441                json!({}),
9442                Some(json!(["/roles/~2/tools"])),
9443            ),
9444            (
9445                "unknown_capabilities_field".to_string(),
9446                "capabilities.future".to_string(),
9447                "<array>".to_string(),
9448                json!({ "future": [] }),
9449                None,
9450            ),
9451        ]);
9452
9453        for (index, (name, field, value, capabilities, runtime_computed)) in
9454            cases.into_iter().enumerate()
9455        {
9456            let registry = Arc::new(Registry::default());
9457            let handler = ControlHandler::new(Arc::clone(&registry));
9458            let response = handler
9459                .handle_control(
9460                    ConnectionId::new((index + 1) as u64),
9461                    capability_grammar_hello_frame(
9462                        capabilities,
9463                        runtime_computed,
9464                        index as u64 + 1,
9465                    ),
9466                )
9467                .expect("invalid HELLO returns a refusal");
9468
9469            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9470            let error = parse_error(&response[0]);
9471            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9472            let message = error["message"]
9473                .as_str()
9474                .expect("error message is a string");
9475            assert!(
9476                message.contains(&field),
9477                "{name}: field missing from {message}"
9478            );
9479            assert!(
9480                message.contains(&value),
9481                "{name}: value missing from {message}"
9482            );
9483            assert_eq!(
9484                registry
9485                    .active_registration_count()
9486                    .expect("registry reads"),
9487                0,
9488                "{name}: refused HELLO must not create a catalog entry"
9489            );
9490        }
9491    }
9492
9493    #[test]
9494    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9495        let registry = Arc::new(Registry::default());
9496        let handler = ControlHandler::new(Arc::clone(&registry));
9497        let capabilities = json!({
9498            "provides": ["credentials-provider/v1"],
9499            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9500            "must_never_reach": ["federation-transport/v1"]
9501        });
9502        let response = handler
9503            .handle_control(
9504                ConnectionId::new(99),
9505                capability_grammar_hello_frame(
9506                    capabilities.clone(),
9507                    Some(json!(["/roles/0/tools"])),
9508                    99,
9509                ),
9510            )
9511            .expect("valid HELLO registers");
9512        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9513
9514        let request = Frame::build(
9515            FrameType::Request,
9516            control_flags(),
9517            0,
9518            0,
9519            100,
9520            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9521                .expect("catalog request serializes"),
9522        )
9523        .expect("catalog request frame builds");
9524        let response = handler
9525            .handle_catalog_list(request, None)
9526            .expect("catalog list succeeds");
9527        let ClientControlResponse::CatalogList { modules, .. } =
9528            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9529        else {
9530            panic!("catalog request must return catalog.list");
9531        };
9532        assert_eq!(modules.len(), 1);
9533        assert_eq!(
9534            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9535            capabilities
9536        );
9537    }
9538
9539    #[test]
9540    fn catalog_list_mirrors_management_operation_description() {
9541        let registry = Arc::new(Registry::default());
9542        let handler = ControlHandler::new(Arc::clone(&registry));
9543        let description = "List managed records and return their identifiers and metadata.";
9544        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9545        manifest.provides = vec![ProviderRole::ManagementSurface {
9546            operations: vec![ManagementOperation {
9547                name: "records.list".to_string(),
9548                kind: ManagementOperationKind::Query,
9549                description: Some(description.to_string()),
9550            }],
9551            config_schema: json!({"type": "object"}),
9552            observability: vec![ObservabilitySurface {
9553                name: "records.stats".to_string(),
9554                kind: ObservabilityKind::Snapshot,
9555            }],
9556            identity_scope: vec![IdentityScope::Project],
9557            concurrency: Concurrency::ModuleManaged,
9558        }];
9559        registry
9560            .register_with_control_ops(
9561                manifest,
9562                PROTOCOL_VERSION,
9563                ConnectionId::new(99),
9564                Vec::new(),
9565            )
9566            .expect("described management manifest registers");
9567
9568        let request = Frame::build(
9569            FrameType::Request,
9570            control_flags(),
9571            0,
9572            0,
9573            100,
9574            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9575                .expect("catalog request serializes"),
9576        )
9577        .expect("catalog request frame builds");
9578        let response = handler
9579            .handle_catalog_list(request, None)
9580            .expect("catalog list succeeds");
9581        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9582        assert_eq!(
9583            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9584            "catalog.list must preserve the declared operation description verbatim"
9585        );
9586    }
9587
9588    #[test]
9589    fn catalog_list_mirrors_registered_operation_delivery() {
9590        let registry = Arc::new(Registry::default());
9591        let handler = ControlHandler::new(Arc::clone(&registry));
9592        let delivery =
9593            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9594        let mut body = serde_json::to_value(ModuleHelloBody {
9595            manifest: manifest("watch-management", PROTOCOL_VERSION),
9596            protocol_ver: PROTOCOL_VERSION,
9597            control_ops: None,
9598            launch_nonce: None,
9599        })
9600        .unwrap();
9601        body["manifest"]["provides"] = json!([{
9602            "role": "management_surface",
9603            "operations": [
9604                {"name": "status", "kind": "query"},
9605                {"name": "future", "kind": "query"},
9606                {"name": "reset", "kind": "mutate"}
9607            ],
9608            "config_schema": {}, "observability": [], "identity_scope": []
9609        }]);
9610        body["manifest"]["operation_delivery"] = delivery.clone();
9611        let hello = Frame::build(
9612            FrameType::Hello,
9613            control_flags(),
9614            0,
9615            0,
9616            99,
9617            serde_json::to_vec(&body).unwrap(),
9618        )
9619        .unwrap();
9620        let response = handler
9621            .handle_control(ConnectionId::new(99), hello)
9622            .unwrap();
9623        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9624
9625        let (_, registrations) = registry.list_modules().unwrap();
9626        let registered = &registrations[0].manifest;
9627        assert!(registered.is_watch_operation("status"));
9628        assert!(!registered.is_watch_operation("future"));
9629        assert!(!registered.is_watch_operation("reset"));
9630        assert!(!registered.is_watch_operation("missing"));
9631
9632        let request = Frame::build(
9633            FrameType::Request,
9634            control_flags(),
9635            0,
9636            0,
9637            100,
9638            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9639        )
9640        .unwrap();
9641        let response = handler.handle_catalog_list(request, None).unwrap();
9642        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9643        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9644        let ClientControlResponse::CatalogList { modules, .. } =
9645            serde_json::from_slice(&response[0].body).unwrap()
9646        else {
9647            panic!("expected catalog.list");
9648        };
9649        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9650    }
9651
9652    #[test]
9653    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9654        let registry = Arc::new(Registry::default());
9655        let supervisor = SupervisorHandle::new();
9656        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9657        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9658        let descriptor = json!({
9659            "schema": "ckios.richui.manifest/1",
9660            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9661            "bytes": 4096,
9662            "chunks": 4
9663        });
9664        let connection = ConnectionId::new(201);
9665        let hello = handler
9666            .handle_control(
9667                connection,
9668                hello_frame_with_rich_ui(
9669                    "vault",
9670                    PROTOCOL_VERSION,
9671                    201,
9672                    Some("the-real-nonce"),
9673                    descriptor.clone(),
9674                ),
9675            )
9676            .unwrap();
9677        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9678
9679        let request =
9680            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9681        let provides = registry
9682            .get_module_by_connection(connection)
9683            .unwrap()
9684            .unwrap()
9685            .manifest
9686            .provides;
9687        handler
9688            .handle_catalog_update(connection, request, provides, None, Some(false))
9689            .unwrap();
9690
9691        let request = Frame::build(
9692            FrameType::Request,
9693            control_flags(),
9694            0,
9695            0,
9696            203,
9697            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9698        )
9699        .unwrap();
9700        let response = handler.handle_catalog_list(request, None).unwrap();
9701        let ClientControlResponse::CatalogList { modules, .. } =
9702            serde_json::from_slice(&response[0].body).unwrap()
9703        else {
9704            panic!("expected catalog.list");
9705        };
9706        assert_eq!(
9707            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9708            descriptor
9709        );
9710        assert!(modules[0].nonce_verified);
9711    }
9712
9713    fn hello_with_role_declarations(declarations: Value) -> Frame {
9714        let mut body = serde_json::to_value(ModuleHelloBody {
9715            manifest: manifest("role-test", PROTOCOL_VERSION),
9716            protocol_ver: PROTOCOL_VERSION,
9717            control_ops: None,
9718            launch_nonce: None,
9719        })
9720        .unwrap();
9721        body["manifest"]["role_declarations"] = declarations;
9722        Frame::build(
9723            FrameType::Hello,
9724            control_flags(),
9725            0,
9726            0,
9727            301,
9728            serde_json::to_vec(&body).unwrap(),
9729        )
9730        .unwrap()
9731    }
9732
9733    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9734        let request = Frame::build(
9735            FrameType::Request,
9736            control_flags(),
9737            0,
9738            0,
9739            302,
9740            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9741        )
9742        .unwrap();
9743        let response = handler.handle_catalog_list(request, None).unwrap();
9744        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9745        let ClientControlResponse::CatalogList { mut modules, .. } =
9746            serde_json::from_slice(&response[0].body).unwrap()
9747        else {
9748            panic!("expected catalog.list");
9749        };
9750        (raw["modules"][0].clone(), modules.remove(0))
9751    }
9752
9753    #[test]
9754    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9755        let registry = Arc::new(Registry::default());
9756        let handler = ControlHandler::new(Arc::clone(&registry));
9757        let declarations = json!({
9758            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9759            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9760        });
9761        let connection = ConnectionId::new(301);
9762        let response = handler
9763            .handle_control(
9764                connection,
9765                hello_with_role_declarations(declarations.clone()),
9766            )
9767            .unwrap();
9768        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9769        let registered = registry
9770            .get_module_by_connection(connection)
9771            .unwrap()
9772            .unwrap()
9773            .manifest;
9774        // Compared as JSON values, not bytes: object key order is not part of
9775        // the relay's promise, and workspace builds unify serde_json's
9776        // preserve_order feature, which changes the order a literal keeps.
9777        assert_eq!(
9778            serde_json::to_value(&registered.role_declarations).unwrap(),
9779            declarations
9780        );
9781        let (raw, entry) = role_declarations_catalog(&handler);
9782        assert_eq!(raw["role_declarations"], declarations);
9783        assert_eq!(entry.role_declarations, registered.role_declarations);
9784
9785        let request =
9786            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9787        handler
9788            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9789            .unwrap();
9790        let (raw, entry) = role_declarations_catalog(&handler);
9791        assert_eq!(raw["role_declarations"], declarations);
9792        assert_eq!(entry.role_declarations, registered.role_declarations);
9793    }
9794
9795    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9796        let registry = Arc::new(Registry::default());
9797        let handler = ControlHandler::new(Arc::clone(&registry));
9798        let capture = EventCapture::default();
9799        let _subscriber =
9800            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9801        let response = handler
9802            .handle_control(
9803                ConnectionId::new(301),
9804                hello_with_role_declarations(declarations),
9805            )
9806            .unwrap();
9807        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9808        assert!(registry
9809            .get_module_by_connection(ConnectionId::new(301))
9810            .unwrap()
9811            .unwrap()
9812            .manifest
9813            .role_declarations
9814            .is_empty());
9815        let (raw, entry) = role_declarations_catalog(&handler);
9816        assert!(raw.get("role_declarations").is_none());
9817        assert!(entry.role_declarations.is_empty());
9818        let warnings: Vec<_> = capture
9819            .events()
9820            .into_iter()
9821            .filter(|event| {
9822                event.fields.get("message").map(String::as_str)
9823                    == Some("dropping whole role_declarations map from HELLO")
9824            })
9825            .collect();
9826        assert_eq!(warnings.len(), 1);
9827        assert_eq!(warnings[0].level, tracing::Level::WARN);
9828        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9829        assert_eq!(warnings[0].fields["reason"], reason);
9830    }
9831
9832    #[test]
9833    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9834        for value in [
9835            json!(null),
9836            json!(false),
9837            json!(0),
9838            json!("opaque"),
9839            json!([]),
9840        ] {
9841            assert_hello_drops_role_declarations_with_warning(
9842                json!({"routing/v1": {}, "broken/v1": value}),
9843                "role_declarations values must be objects",
9844            );
9845        }
9846        assert_hello_drops_role_declarations_with_warning(
9847            json!([]),
9848            "role_declarations must be an object",
9849        );
9850    }
9851
9852    #[test]
9853    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9854        assert_hello_drops_role_declarations_with_warning(
9855            json!({"routing/v1": {}, "bad key": {}}),
9856            "role_declarations key must use capability-name grammar",
9857        );
9858    }
9859
9860    #[test]
9861    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9862        assert_hello_drops_role_declarations_with_warning(
9863            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9864            "role_declarations exceeds 16384 compact JSON bytes",
9865        );
9866    }
9867
9868    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9869        let mut body = serde_json::to_value(ModuleHelloBody {
9870            manifest: manifest("flow-ops", PROTOCOL_VERSION),
9871            protocol_ver: PROTOCOL_VERSION,
9872            control_ops: None,
9873            launch_nonce: None,
9874        })
9875        .unwrap();
9876        body["manifest"]["provides"] = json!([{
9877            "role": "management_surface",
9878            "operations": [
9879                {"name": "diagnostics.findings", "kind": "query"},
9880                {"name": "reset", "kind": "mutate"}
9881            ],
9882            "config_schema": {}, "observability": [], "identity_scope": []
9883        }]);
9884        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9885        if let Some(flow_operations) = flow_operations {
9886            body["manifest"]["flow_operations"] = flow_operations;
9887        }
9888        Frame::build(
9889            FrameType::Hello,
9890            control_flags(),
9891            0,
9892            0,
9893            401,
9894            serde_json::to_vec(&body).unwrap(),
9895        )
9896        .unwrap()
9897    }
9898
9899    #[test]
9900    fn catalog_list_mirrors_flow_operations_unchanged() {
9901        // Order is kept as declared (not sorted) and an empty list stays an
9902        // affirmative "none", distinct from an absent declaration.
9903        for (capability, flow_operations) in [
9904            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9905            ("call-modes/v1", json!(["diagnostics.findings"])),
9906            ("flow-scopes/v1", json!([])),
9907        ] {
9908            let registry = Arc::new(Registry::default());
9909            let handler = ControlHandler::new(Arc::clone(&registry));
9910            let connection = ConnectionId::new(401);
9911            let response = handler
9912                .handle_control(
9913                    connection,
9914                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9915                )
9916                .unwrap();
9917            assert_eq!(response[0].header.ty, FrameType::HelloAck);
9918            let registered = registry
9919                .get_module_by_connection(connection)
9920                .unwrap()
9921                .unwrap()
9922                .manifest;
9923            assert_eq!(
9924                serde_json::to_value(&registered.flow_operations).unwrap(),
9925                flow_operations
9926            );
9927            let (raw, entry) = role_declarations_catalog(&handler);
9928            assert_eq!(raw["flow_operations"], flow_operations);
9929            assert_eq!(entry.flow_operations, registered.flow_operations);
9930        }
9931
9932        let registry = Arc::new(Registry::default());
9933        let handler = ControlHandler::new(Arc::clone(&registry));
9934        let response = handler
9935            .handle_control(
9936                ConnectionId::new(401),
9937                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9938            )
9939            .unwrap();
9940        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9941        let (raw, entry) = role_declarations_catalog(&handler);
9942        assert!(raw.get("flow_operations").is_none());
9943        assert_eq!(entry.flow_operations, None);
9944    }
9945
9946    fn assert_hello_drops_flow_operations_with_warning(
9947        capabilities: Value,
9948        flow_operations: Value,
9949        reason: &str,
9950    ) {
9951        let registry = Arc::new(Registry::default());
9952        let handler = ControlHandler::new(Arc::clone(&registry));
9953        let capture = EventCapture::default();
9954        let _subscriber =
9955            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9956        let response = handler
9957            .handle_control(
9958                ConnectionId::new(401),
9959                hello_with_flow_operations(capabilities, Some(flow_operations)),
9960            )
9961            .unwrap();
9962        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9963        let registered = registry
9964            .get_module_by_connection(ConnectionId::new(401))
9965            .unwrap()
9966            .expect("the module still registers")
9967            .manifest;
9968        assert_eq!(registered.module_id, "flow-ops");
9969        assert_eq!(registered.flow_operations, None);
9970        let (raw, entry) = role_declarations_catalog(&handler);
9971        assert_eq!(raw["module_id"], "flow-ops");
9972        assert!(raw.get("flow_operations").is_none());
9973        assert_eq!(entry.flow_operations, None);
9974        let warnings: Vec<_> = capture
9975            .events()
9976            .into_iter()
9977            .filter(|event| {
9978                event.fields.get("message").map(String::as_str)
9979                    == Some("dropping flow_operations from HELLO")
9980            })
9981            .collect();
9982        assert_eq!(warnings.len(), 1);
9983        assert_eq!(warnings[0].level, tracing::Level::WARN);
9984        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9985        assert_eq!(warnings[0].fields["reason"], reason);
9986    }
9987
9988    #[test]
9989    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9990        assert_hello_drops_flow_operations_with_warning(
9991            json!(["flow-scopes/v1"]),
9992            json!(["diagnostics.findings", "missing"]),
9993            "flow_operations names 'missing', which is not a declared management operation",
9994        );
9995    }
9996
9997    #[test]
9998    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9999        assert_hello_drops_flow_operations_with_warning(
10000            json!([]),
10001            json!(["diagnostics.findings"]),
10002            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10003        );
10004        // A run-only capability does not admit flows either.
10005        assert_hello_drops_flow_operations_with_warning(
10006            json!(["agent-run-scopes/v1"]),
10007            json!([]),
10008            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10009        );
10010    }
10011
10012    #[test]
10013    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10014        assert_hello_drops_flow_operations_with_warning(
10015            json!(["flow-scopes/v1"]),
10016            json!(["reset", "reset"]),
10017            "flow_operations names 'reset' more than once",
10018        );
10019        assert_hello_drops_flow_operations_with_warning(
10020            json!(["flow-scopes/v1"]),
10021            json!(vec!["reset"; 257]),
10022            "flow_operations lists 257 names, more than 256",
10023        );
10024        assert_hello_drops_flow_operations_with_warning(
10025            json!(["flow-scopes/v1"]),
10026            json!("reset"),
10027            "flow_operations must be an array of operation names",
10028        );
10029    }
10030
10031    #[test]
10032    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10033        let registry = Arc::new(Registry::default());
10034        let handler = ControlHandler::new(registry.clone());
10035        let descriptor = json!({
10036            "schema": "ckios.richui.manifest/1",
10037            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10038            "bytes": 128,
10039            "chunks": 2
10040        });
10041        let malformed = [
10042            ("schema", json!("")),
10043            ("schema", json!("a".repeat(65))),
10044            ("schema", json!("Bad schema")),
10045            (
10046                "manifest_digest",
10047                json!(format!("sha256:{}", "A".repeat(64))),
10048            ),
10049            ("manifest_digest", json!("sha256:abc")),
10050            ("bytes", json!(0)),
10051            ("bytes", json!(16_777_217)),
10052            ("bytes", json!("128")),
10053            ("chunks", json!(0)),
10054            ("chunks", json!(1025)),
10055            ("chunks", json!("2")),
10056        ];
10057        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10058            let mut invalid = descriptor.clone();
10059            invalid[field] = invalid_value;
10060            let response = handler
10061                .handle_control(
10062                    ConnectionId::new(210 + index as u64),
10063                    hello_frame_with_rich_ui(
10064                        &format!("bad-rich-ui-{index}"),
10065                        PROTOCOL_VERSION,
10066                        210 + index as u64,
10067                        None,
10068                        invalid,
10069                    ),
10070                )
10071                .unwrap();
10072            assert_eq!(response[0].header.ty, FrameType::Error);
10073            let error = parse_error(&response[0]);
10074            assert_eq!(error["code"], "invalid_manifest");
10075            assert!(error["message"]
10076                .as_str()
10077                .unwrap()
10078                .contains(&format!("rich_ui.{field}")));
10079            assert!(registry
10080                .get_module(&format!("bad-rich-ui-{index}"))
10081                .unwrap()
10082                .is_none());
10083        }
10084    }
10085
10086    #[test]
10087    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10088        let registry = Arc::new(Registry::default());
10089        let handler = ControlHandler::new(Arc::clone(&registry));
10090        let connection = ConnectionId::new(219);
10091        registry
10092            .register_with_control_ops(
10093                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10094                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10095                        "bad schema",
10096                        format!("sha256:{}", "a".repeat(64)),
10097                        128,
10098                        2,
10099                    )))
10100                    .build(),
10101                PROTOCOL_VERSION,
10102                connection,
10103                Vec::new(),
10104            )
10105            .unwrap();
10106        let request =
10107            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10108        let response = handler
10109            .handle_catalog_update(connection, request, Vec::new(), None, None)
10110            .unwrap();
10111        assert_eq!(response[0].header.ty, FrameType::Error);
10112        let error = parse_error(&response[0]);
10113        assert_eq!(error["code"], "invalid_manifest");
10114        assert!(error["message"]
10115            .as_str()
10116            .unwrap()
10117            .contains("rich_ui.schema"));
10118    }
10119
10120    #[test]
10121    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10122        let registry = Arc::new(Registry::default());
10123        let supervisor = SupervisorHandle::new();
10124        supervisor.open_swap("provider", "candidate-nonce".to_string());
10125        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10126        let connection = ConnectionId::new(218);
10127        let hello = handler
10128            .handle_control(
10129                connection,
10130                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10131            )
10132            .unwrap();
10133        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10134        assert!(registry.list_modules().unwrap().1.is_empty());
10135
10136        registry.promote_candidate("provider").unwrap().unwrap();
10137        let request = Frame::build(
10138            FrameType::Request,
10139            control_flags(),
10140            0,
10141            0,
10142            219,
10143            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10144        )
10145        .unwrap();
10146        let response = handler.handle_catalog_list(request, None).unwrap();
10147        let ClientControlResponse::CatalogList { modules, .. } =
10148            serde_json::from_slice(&response[0].body).unwrap()
10149        else {
10150            panic!("expected catalog.list");
10151        };
10152        assert_eq!(modules.len(), 1);
10153        assert!(modules[0].nonce_verified);
10154    }
10155
10156    #[test]
10157    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10158        let registry = Arc::new(Registry::default());
10159        let handler = ControlHandler::new(Arc::clone(&registry));
10160        let hello = Frame::build(
10161            FrameType::Hello,
10162            control_flags(),
10163            0,
10164            0,
10165            220,
10166            serde_json::to_vec(&ModuleHelloBody {
10167                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10168                protocol_ver: PROTOCOL_VERSION,
10169                control_ops: None,
10170                launch_nonce: Some("unverified".to_string()),
10171            })
10172            .unwrap(),
10173        )
10174        .unwrap();
10175        assert_eq!(
10176            handler
10177                .handle_control(ConnectionId::new(220), hello)
10178                .unwrap()[0]
10179                .header
10180                .ty,
10181            FrameType::HelloAck
10182        );
10183        let request = Frame::build(
10184            FrameType::Request,
10185            control_flags(),
10186            0,
10187            0,
10188            221,
10189            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10190        )
10191        .unwrap();
10192        let response = handler.handle_catalog_list(request, None).unwrap();
10193        let ClientControlResponse::CatalogList { modules, .. } =
10194            serde_json::from_slice(&response[0].body).unwrap()
10195        else {
10196            panic!("expected catalog.list");
10197        };
10198        assert_eq!(
10199            serde_json::to_vec(&modules[0]).unwrap(),
10200            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10201        );
10202        assert!(!modules[0].nonce_verified);
10203    }
10204
10205    #[test]
10206    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10207        let registry = Arc::new(Registry::default());
10208        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10209            [("vault".to_string(), true), ("squatter".to_string(), true)],
10210            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10211        );
10212        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10213        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10214            provides: vec!["credentials-provider/v1".to_string()],
10215            requires: Vec::new(),
10216            must_never_reach: Vec::new(),
10217        });
10218        let frame = Frame::build(
10219            FrameType::Hello,
10220            control_flags(),
10221            0,
10222            0,
10223            77,
10224            serde_json::to_vec(&ModuleHelloBody {
10225                manifest: squatter,
10226                protocol_ver: PROTOCOL_VERSION,
10227                control_ops: None,
10228                launch_nonce: None,
10229            })
10230            .expect("HELLO serializes"),
10231        )
10232        .expect("HELLO frame builds");
10233        let response = handler
10234            .handle_control(ConnectionId::new(77), frame)
10235            .expect("reserved claim receives a typed refusal");
10236        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10237        assert_eq!(
10238            registry
10239                .active_registration_count()
10240                .expect("registry reads"),
10241            0,
10242            "a reserved capability refusal must not leave a catalog entry"
10243        );
10244    }
10245
10246    #[test]
10247    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10248        for settlement in ["timeout", "inconclusive", "drop"] {
10249            let breakers = RouteBindBreakers::default();
10250            let RouteBindAdmission::Admitted {
10251                guard: mut old,
10252                probe: false,
10253            } = breakers.admit("prov")
10254            else {
10255                panic!("ordinary relay admitted")
10256            };
10257            let RouteBindAdmission::Admitted {
10258                guard: mut opener, ..
10259            } = breakers.admit("prov")
10260            else {
10261                panic!("second relay admitted")
10262            };
10263            assert!(
10264                !opener
10265                    .record_timeout(1, Duration::ZERO)
10266                    .unwrap()
10267                    .reopened_after_probe
10268            );
10269            let RouteBindAdmission::Admitted {
10270                guard: mut probe,
10271                probe: true,
10272            } = breakers.admit("prov")
10273            else {
10274                panic!("one cooldown probe admitted")
10275            };
10276            match settlement {
10277                "timeout" => assert!(
10278                    !old.record_timeout(1, Duration::ZERO)
10279                        .unwrap()
10280                        .reopened_after_probe
10281                ),
10282                "inconclusive" => old.record_inconclusive(),
10283                "drop" => drop(old),
10284                _ => unreachable!(),
10285            }
10286            assert!(
10287                matches!(
10288                    breakers.admit("prov"),
10289                    RouteBindAdmission::Refused {
10290                        probe_in_flight: true,
10291                        ..
10292                    }
10293                ),
10294                "{settlement} of a pre-open relay cannot release the real probe"
10295            );
10296            assert!(
10297                probe
10298                    .record_timeout(1, Duration::ZERO)
10299                    .unwrap()
10300                    .reopened_after_probe
10301            );
10302            assert!(matches!(
10303                breakers.admit("prov"),
10304                RouteBindAdmission::Admitted { probe: true, .. }
10305            ));
10306        }
10307        let breakers = RouteBindBreakers::default();
10308        let admit = || match breakers.admit("prov") {
10309            RouteBindAdmission::Admitted { guard, .. } => guard,
10310            _ => panic!("relay admitted"),
10311        };
10312        admit().record_timeout(1, Duration::ZERO);
10313        let mut old_probe = admit();
10314        breakers.reset_for_new_module_connection("prov");
10315        admit().record_timeout(1, Duration::ZERO);
10316        let _new_probe = admit();
10317        old_probe.record_inconclusive();
10318        assert!(matches!(
10319            breakers.admit("prov"),
10320            RouteBindAdmission::Refused {
10321                probe_in_flight: true,
10322                ..
10323            }
10324        ));
10325    }
10326
10327    #[tokio::test]
10328    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10329        for candidate in [false, true] {
10330            let registry = Arc::new(Registry::default());
10331            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10332                [("vault".to_string(), true), ("squatter".to_string(), true)],
10333                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10334            );
10335            let conn = ConnectionId::new(77);
10336            let (ctx, mut rx) = route_ctx(conn);
10337            let initial = capability_manifest("squatter", &[], &[]);
10338            if candidate {
10339                registry
10340                    .register_candidate_with_control_ops(
10341                        initial.clone(),
10342                        PROTOCOL_VERSION,
10343                        conn,
10344                        module_baseline_control_ops(),
10345                    )
10346                    .unwrap();
10347                handler
10348                    .forwarding
10349                    .register_candidate_module_connection(
10350                        conn,
10351                        "squatter".to_string(),
10352                        PROTOCOL_VERSION,
10353                        manifest_concurrency(&initial),
10354                        ctx.egress.clone(),
10355                    )
10356                    .unwrap();
10357            } else {
10358                hello_via_sink(
10359                    &handler,
10360                    &ctx,
10361                    &mut rx,
10362                    hello_frame_with_manifest(initial.clone(), 1),
10363                )
10364                .await;
10365            }
10366            let response = handler
10367                .handle_control_frame(
10368                    &ctx,
10369                    catalog_update_with_capabilities_frame(
10370                        2,
10371                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
10372                            .capabilities
10373                            .unwrap(),
10374                    ),
10375                )
10376                .await
10377                .unwrap();
10378            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10379            assert_eq!(
10380                registry
10381                    .get_module_by_connection(conn)
10382                    .unwrap()
10383                    .unwrap()
10384                    .manifest,
10385                initial
10386            );
10387        }
10388    }
10389
10390    #[test]
10391    fn server_describe_surfaces_required_capability_verdict_fields() {
10392        let registry = Arc::new(Registry::default());
10393        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10394            [
10395                ("consumer".to_string(), true),
10396                ("provider".to_string(), false),
10397            ],
10398            BTreeMap::new(),
10399        );
10400        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10401        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10402            provides: Vec::new(),
10403            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10404                capability: "credentials-provider/v1".to_string(),
10405                need: subc_protocol::manifest::CapabilityNeed::Required,
10406            }],
10407            must_never_reach: Vec::new(),
10408        });
10409        let hello = Frame::build(
10410            FrameType::Hello,
10411            control_flags(),
10412            0,
10413            0,
10414            78,
10415            serde_json::to_vec(&ModuleHelloBody {
10416                manifest: consumer,
10417                protocol_ver: PROTOCOL_VERSION,
10418                control_ops: None,
10419                launch_nonce: None,
10420            })
10421            .expect("HELLO serializes"),
10422        )
10423        .expect("HELLO frame builds");
10424        handler
10425            .handle_control(ConnectionId::new(78), hello)
10426            .expect("consumer registers");
10427        let describe = Frame::build(
10428            FrameType::Request,
10429            control_flags(),
10430            0,
10431            0,
10432            79,
10433            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10434                .expect("request serializes"),
10435        )
10436        .expect("describe frame builds");
10437        let response = handler
10438            .handle_server_describe(describe)
10439            .expect("server.describe succeeds");
10440        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10441        let requirement = &rendered["capability_requirements"][0];
10442        assert_eq!(requirement["consumer"], "consumer");
10443        assert_eq!(requirement["verdict"], "never_provided");
10444        assert_eq!(requirement["episode_seq"], 1);
10445        assert_eq!(requirement["config_satisfiable"], false);
10446        assert_eq!(requirement["runtime_available"], false);
10447        assert!(requirement["detail"]
10448            .as_str()
10449            .expect("detail string")
10450            .contains("credentials-provider/v1"));
10451    }
10452
10453    /// Register a consumer that requires `project-identity/v1` and the provider
10454    /// of it, optionally mark daemon shutdown as begun, then deregister the
10455    /// provider as its connection closes. Returns the `capability.requirement`
10456    /// events that deregistration logged and the verdict the evaluator then
10457    /// holds for the requirement (the one `route.open` refusals read).
10458    fn deregister_required_provider(
10459        daemon_shutdown_begun: bool,
10460    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10461        const CAPABILITY: &str = "project-identity/v1";
10462        let registry = Arc::new(Registry::default());
10463        let daemon_shutdown = DaemonShutdownFlag::default();
10464        let handler = ControlHandler::new(Arc::clone(&registry))
10465            .with_capability_config(
10466                [
10467                    ("consumer".to_string(), true),
10468                    ("provider".to_string(), true),
10469                ],
10470                BTreeMap::new(),
10471            )
10472            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10473        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10474        consumer.capabilities = Some(CapabilityDeclarations {
10475            provides: Vec::new(),
10476            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10477                capability: CAPABILITY.to_string(),
10478                need: subc_protocol::manifest::CapabilityNeed::Required,
10479            }],
10480            must_never_reach: Vec::new(),
10481        });
10482        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10483        let provider_conn = ConnectionId::new(182);
10484        for (conn, manifest, corr) in [
10485            (ConnectionId::new(181), consumer, 181),
10486            (provider_conn, provider, 182),
10487        ] {
10488            let hello = handler
10489                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10490                .expect("module registers");
10491            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10492        }
10493        assert_eq!(
10494            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10495            Some(CapabilityVerdict::Provided),
10496            "the registered provider must satisfy the requirement first"
10497        );
10498        if daemon_shutdown_begun {
10499            daemon_shutdown.set_for_test();
10500        }
10501
10502        let capture = EventCapture::default();
10503        let _subscriber =
10504            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10505        handler
10506            .cleanup_connection(provider_conn)
10507            .expect("provider deregisters");
10508        let events = capture
10509            .events()
10510            .into_iter()
10511            .filter(|event| {
10512                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10513            })
10514            .collect();
10515        (
10516            events,
10517            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10518        )
10519    }
10520
10521    #[test]
10522    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10523        let (events, verdict) = deregister_required_provider(true);
10524        assert_eq!(
10525            verdict,
10526            Some(CapabilityVerdict::NeverProvided),
10527            "shutdown must not change the verdict route.open refusals read"
10528        );
10529        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10530        let event = &events[0];
10531        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10532        assert_eq!(
10533            event.fields.get("cause").map(String::as_str),
10534            Some("\"daemon_shutdown\""),
10535            "{event:?}"
10536        );
10537        assert_eq!(
10538            event.fields.get("verdict").map(String::as_str),
10539            Some("\"never_provided\""),
10540            "{event:?}"
10541        );
10542    }
10543
10544    #[test]
10545    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10546        let (events, verdict) = deregister_required_provider(false);
10547        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10548        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10549        let event = &events[0];
10550        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10551        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10552        assert_eq!(
10553            event.fields.get("verdict").map(String::as_str),
10554            Some("\"never_provided\""),
10555            "{event:?}"
10556        );
10557    }
10558
10559    #[test]
10560    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10561        let registry = Arc::new(Registry::default());
10562        let handler = ControlHandler::new(Arc::clone(&registry));
10563        let hello = handler
10564            .handle_control(
10565                ConnectionId::new(101),
10566                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10567            )
10568            .expect("legacy HELLO registers");
10569        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10570
10571        let request = Frame::build(
10572            FrameType::Request,
10573            control_flags(),
10574            0,
10575            0,
10576            102,
10577            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10578                .expect("catalog request serializes"),
10579        )
10580        .expect("catalog request frame builds");
10581        let response = handler
10582            .handle_catalog_list(request, None)
10583            .expect("catalog list succeeds");
10584        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10585        assert!(
10586            body["modules"][0].get("capabilities").is_none(),
10587            "legacy manifest must retain an absent capabilities field on catalog.list"
10588        );
10589    }
10590
10591    #[test]
10592    fn hello_ack_omits_storage_when_no_storage_config() {
10593        let registry = Arc::new(Registry::default());
10594        let handler = ControlHandler::new(Arc::clone(&registry));
10595        let responses = handler
10596            .handle_control(
10597                ConnectionId::new(1),
10598                hello_frame("aft", PROTOCOL_VERSION, 7),
10599            )
10600            .unwrap();
10601        let ack = parse_ack(&responses[0]);
10602        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10603        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10604    }
10605
10606    #[tokio::test]
10607    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10608        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10609        let registry = Arc::new(Registry::default());
10610        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10611        let responses = handler
10612            .handle_control(
10613                ConnectionId::new(1),
10614                hello_frame("aft", PROTOCOL_VERSION, 7),
10615            )
10616            .unwrap();
10617        let ack = parse_ack(&responses[0]);
10618        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10619
10620        let described = handler
10621            .handle_control_frame(
10622                &route_ctx(ConnectionId::new(2)).0,
10623                Frame::build(
10624                    FrameType::Request,
10625                    control_flags(),
10626                    0,
10627                    0,
10628                    9,
10629                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10630                )
10631                .unwrap(),
10632            )
10633            .await
10634            .unwrap();
10635        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10636            serde_json::from_slice(&described[0].body).unwrap()
10637        else {
10638            panic!("server.describe answered with another shape");
10639        };
10640        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10641    }
10642
10643    #[test]
10644    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10645        // With a central sqlite storage policy, each registering module gets its
10646        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10647        let registry = Arc::new(Registry::default());
10648        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10649            crate::daemon_config::StorageConfig::Sqlite {
10650                data_home: std::path::PathBuf::from("/data"),
10651            },
10652        ));
10653
10654        let responses = handler
10655            .handle_control(
10656                ConnectionId::new(1),
10657                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10658            )
10659            .unwrap();
10660        let ack = parse_ack(&responses[0]);
10661        assert_eq!(
10662            ack.storage,
10663            Some(serde_json::json!({
10664                "module_id": "alfonso-routing",
10665                "storage_namespace": "default",
10666                "isolation": { "kind": "module" },
10667                "backend": {
10668                    "backend": "sqlite",
10669                    "path": "/data/cortexkit/alfonso-routing/store.db"
10670                }
10671            })),
10672            "the delivered descriptor is the module's own sqlite store path"
10673        );
10674    }
10675
10676    #[test]
10677    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10678        let registry = Arc::new(Registry::default());
10679        let handler = ControlHandler::new(Arc::clone(&registry));
10680        let conn = ConnectionId::new(1);
10681        let responses = handler
10682            .handle_control(
10683                conn,
10684                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10685            )
10686            .unwrap();
10687        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10688        let registration = registry.get_module("aft").unwrap().unwrap();
10689        assert_eq!(registration.control_ops, module_baseline_control_ops());
10690
10691        let frame =
10692            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10693        assert!(handler
10694            .guard_module_control_op(&frame, "aft", "route.bind")
10695            .unwrap()
10696            .is_none());
10697        let error = handler
10698            .guard_module_control_op(&frame, "aft", "test.synthetic")
10699            .unwrap()
10700            .expect("synthetic ungranted op should be rejected");
10701        assert_eq!(error.header.ty, FrameType::Error);
10702        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10703    }
10704
10705    #[test]
10706    fn hello_control_ops_some_adds_optional_grants() {
10707        let registry = Arc::new(Registry::default());
10708        let handler = ControlHandler::new(Arc::clone(&registry));
10709        handler
10710            .handle_control(
10711                ConnectionId::new(1),
10712                hello_frame_with_control_ops(
10713                    "aft",
10714                    PROTOCOL_VERSION,
10715                    7,
10716                    Some(vec![
10717                        "future.synthetic".to_string(),
10718                        "route.bind".to_string(),
10719                    ]),
10720                ),
10721            )
10722            .unwrap();
10723        let registration = registry.get_module("aft").unwrap().unwrap();
10724        assert_eq!(
10725            registration.control_ops,
10726            vec![
10727                "route.bind".to_string(),
10728                "route.status".to_string(),
10729                "future.synthetic".to_string(),
10730            ]
10731        );
10732        let frame =
10733            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10734        assert!(handler
10735            .guard_module_control_op(&frame, "aft", "future.synthetic")
10736            .unwrap()
10737            .is_none());
10738    }
10739
10740    #[tokio::test]
10741    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10742        let registry = Arc::new(Registry::default());
10743        let forwarding = Arc::new(ForwardingTable::default());
10744        let handler =
10745            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10746        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10747        hello_via_sink(
10748            &handler,
10749            &module_ctx,
10750            &mut module_rx,
10751            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10752        )
10753        .await;
10754
10755        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10756        let responses = handler
10757            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10758            .await
10759            .unwrap();
10760        assert_eq!(responses.len(), 1);
10761        assert_eq!(responses[0].header.ty, FrameType::Error);
10762        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10763        assert!(module_rx.try_recv().is_err());
10764    }
10765
10766    #[tokio::test]
10767    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10768        let registry = Arc::new(Registry::default());
10769        let forwarding = Arc::new(ForwardingTable::default());
10770        let handler =
10771            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10772        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10773        hello_via_sink(
10774            &handler,
10775            &module_ctx,
10776            &mut module_rx,
10777            hello_frame_with_control_ops(
10778                "aft",
10779                PROTOCOL_VERSION,
10780                7,
10781                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10782            ),
10783        )
10784        .await;
10785
10786        let project_root = unique_project_root("demux");
10787        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10788        let route_handler = handler.clone();
10789        let route_task = tokio::spawn(async move {
10790            route_handler
10791                .handle_control_frame(
10792                    &route_client_ctx,
10793                    route_open_frame(100, "aft", project_root),
10794                )
10795                .await
10796                .unwrap()
10797        });
10798        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10799            .await
10800            .unwrap()
10801            .unwrap();
10802        assert!(matches!(
10803            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10804            ModuleControlRequest::RouteBind { .. }
10805        ));
10806
10807        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10808        let health_handler = handler.clone();
10809        let health_task = tokio::spawn(async move {
10810            health_handler
10811                .handle_control_frame(
10812                    &health_client_ctx,
10813                    supervisor_health_probe_frame(101, "aft"),
10814                )
10815                .await
10816                .unwrap()
10817        });
10818        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10819            .await
10820            .unwrap()
10821            .unwrap();
10822        assert_eq!(
10823            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10824            ModuleControlRequest::HealthCheck {}
10825        );
10826
10827        handler
10828            .handle_control_frame(
10829                &module_ctx,
10830                health_response(health_frame.header.corr, HealthStatus::Degraded),
10831            )
10832            .await
10833            .unwrap();
10834        let health_response = health_task.await.unwrap();
10835        assert_eq!(health_response.len(), 1);
10836        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10837            ClientControlResponse::SupervisorHealthProbe {
10838                module_id,
10839                status,
10840                detail,
10841                metrics,
10842            } => {
10843                assert_eq!(module_id, "aft");
10844                assert_eq!(status, HealthStatus::Degraded);
10845                assert_eq!(detail.as_deref(), Some("warming"));
10846                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10847            }
10848            other => panic!("unexpected health response: {other:?}"),
10849        }
10850
10851        handler
10852            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10853            .await
10854            .unwrap();
10855        let route_response = route_task.await.unwrap();
10856        assert!(route_response.is_empty());
10857        let published = route_client_rx.recv().await.unwrap();
10858        assert!(matches!(
10859            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10860            ClientControlResponse::RouteOpen { .. }
10861        ));
10862    }
10863
10864    /// Start one `route.open` on `client_connection` and return its still-running
10865    /// handler task together with the `route.bind` the module received for it.
10866    /// The handler blocks until the module answers, so it has to run as a task
10867    /// while the test drives the module side.
10868    async fn relay_route_open(
10869        handler: &ControlHandler,
10870        client_connection: ConnectionId,
10871        client_egress: &FrameSink,
10872        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10873        corr: u64,
10874        module_id: &str,
10875        project_root_label: &str,
10876    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10877        let ctx = RouteCtx {
10878            connection_id: client_connection,
10879            egress: client_egress.clone(),
10880        };
10881        let handler = handler.clone();
10882        let project_root = unique_project_root(project_root_label);
10883        let module_id = module_id.to_string();
10884        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10885        let task = tokio::spawn(async move {
10886            let _guard = tracing::dispatcher::set_default(&dispatch);
10887            handler
10888                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10889                .await
10890                .unwrap()
10891        });
10892        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10893            .await
10894            .expect("module receives the relayed route.bind")
10895            .expect("module egress is open");
10896        (task, bind.frame)
10897    }
10898
10899    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10900        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10901            ModuleControlRequest::RouteBind {
10902                route_channel,
10903                epoch,
10904                ..
10905            } => (route_channel, epoch),
10906            other => panic!("expected a route.bind request, got {other:?}"),
10907        }
10908    }
10909
10910    fn published_route(frame: &Frame) -> (u16, u32) {
10911        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10912            ClientControlResponse::RouteOpen {
10913                route_channel,
10914                route_epoch,
10915            } => (route_channel, route_epoch),
10916            other => panic!("expected a route.open response, got {other:?}"),
10917        }
10918    }
10919
10920    /// Reproduction of a production outage. A client had `route.open`s in
10921    /// flight to a module and was already marked closing -- its egress had refused a
10922    /// module frame, so the daemon asked its connection to end -- while its sink
10923    /// was still open. When the module acked those binds, the daemon refused to
10924    /// commit a route for a closing client, and that refusal was returned from
10925    /// the MODULE connection's frame handler, where a router error that has no
10926    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10927    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10928    /// its tools for hours -- one client's teardown took down a connection
10929    /// carrying ~170 other routes.
10930    ///
10931    /// The window is opened here by calling the production path that opens it
10932    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10933    /// state that matters is "in `closing_connections`, sink still open, relay
10934    /// still pending", and it lasts only from the close request until the
10935    /// connection loop reacts to it; a socket-level test can flood a client into
10936    /// that escalation but cannot pin the module's ack inside the window. Closing
10937    /// the socket instead takes the other path entirely -- connection teardown
10938    /// removes the pending relay under the same lock, so the ack finds nothing.
10939    #[tokio::test]
10940    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10941        let registry = Arc::new(Registry::default());
10942        let forwarding = Arc::new(ForwardingTable::default());
10943        let handler =
10944            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10945
10946        let module_connection = ConnectionId::new(30);
10947        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10948        hello_via_sink(
10949            &handler,
10950            &module_ctx,
10951            &mut module_rx,
10952            hello_frame("aft", PROTOCOL_VERSION, 7),
10953        )
10954        .await;
10955
10956        let dying_client = ConnectionId::new(31);
10957        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10958
10959        // A published route on the dying client. The escalation below only marks
10960        // a connection closing for a route it has already published.
10961        let (first_task, first_bind) = relay_route_open(
10962            &handler,
10963            dying_client,
10964            &dying_ctx.egress,
10965            &mut module_rx,
10966            100,
10967            "aft",
10968            "closing-first",
10969        )
10970        .await;
10971        handler
10972            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10973            .await
10974            .unwrap();
10975        assert!(first_task.await.unwrap().is_empty());
10976        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10977
10978        // A second route.open from the same client, relayed and awaiting its ack.
10979        let (second_task, second_bind) = relay_route_open(
10980            &handler,
10981            dying_client,
10982            &dying_ctx.egress,
10983            &mut module_rx,
10984            101,
10985            "aft",
10986            "closing-second",
10987        )
10988        .await;
10989        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10990
10991        // The window: the client is closing, its sink is still open, and its
10992        // second bind is still pending.
10993        assert!(forwarding
10994            .escalate_client_delivery_failure(
10995                dying_client,
10996                first_channel,
10997                first_epoch,
10998                CloseReason::new(
10999                    "module_to_client_delivery_failed",
11000                    "client egress refused a module frame",
11001                ),
11002                crate::forwarding::UndeliveredFrame {
11003                    module_id: None,
11004                    sink: &dying_ctx.egress,
11005                },
11006            )
11007            .unwrap());
11008        assert!(!dying_ctx.egress.is_closed());
11009
11010        // The frame that used to end the module connection.
11011        let ack = handler
11012            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11013            .await;
11014        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11015        if module_loop_error.is_some() {
11016            // What the server's connection loop does with a router error that has
11017            // no ERROR-frame translation: end the connection, which releases the
11018            // module's registration and every route on it.
11019            handler.cleanup_connection(module_connection).unwrap();
11020        }
11021        // Read the module's next frame before opening the co-tenant's route, so
11022        // the GOODBYE assertion below is about THIS ack and not about later
11023        // traffic. `None` means the module was told nothing.
11024        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11025            .await
11026            .ok()
11027            .flatten();
11028
11029        // 1. The module connection is still registered.
11030        assert!(
11031            registry
11032                .get_module_by_connection(module_connection)
11033                .unwrap()
11034                .is_some(),
11035            "one client's closing connection ended the shared module connection: \
11036             {module_loop_error:?}"
11037        );
11038        // ...and still serving: another client can open and use a route on it.
11039        let cotenant = ConnectionId::new(32);
11040        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11041        let (cotenant_task, cotenant_bind) = relay_route_open(
11042            &handler,
11043            cotenant,
11044            &cotenant_ctx.egress,
11045            &mut module_rx,
11046            102,
11047            "aft",
11048            "closing-cotenant",
11049        )
11050        .await;
11051        handler
11052            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11053            .await
11054            .unwrap();
11055        assert!(cotenant_task.await.unwrap().is_empty());
11056        let (cotenant_channel, cotenant_epoch) =
11057            published_route(&cotenant_rx.recv().await.unwrap());
11058        assert!(matches!(
11059            forwarding
11060                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11061                .unwrap(),
11062            DataRoute::Client(DataRouteState::Bound(_))
11063        ));
11064
11065        // 2. The module was told to drop the binding it created for the route
11066        //    that will never be published.
11067        let goodbye = post_ack_module_frame
11068            .expect("module receives a GOODBYE for the abandoned route channel");
11069        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11070        assert_eq!(goodbye.header.channel, abandoned_channel);
11071        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11072
11073        // 3. The dying client received nothing: no route was ever published to
11074        //    it. Its route.open is answered as unavailable, which the connection
11075        //    loop would write to a socket that is already going away.
11076        assert!(dying_rx.try_recv().is_err());
11077        let second_response = second_task.await.unwrap();
11078        assert_eq!(second_response.len(), 1);
11079        assert_eq!(
11080            parse_error(&second_response[0])["code"],
11081            "target_unavailable"
11082        );
11083    }
11084
11085    /// One `catalog.list` as a client sees it: the generation and the listed
11086    /// module ids with their readiness.
11087    async fn catalog_snapshot(
11088        handler: &ControlHandler,
11089        client_ctx: &RouteCtx,
11090        corr: u64,
11091    ) -> (u64, Vec<(String, bool)>) {
11092        let body =
11093            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11094        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11095        let reply = handler
11096            .handle_control_frame(client_ctx, request)
11097            .await
11098            .unwrap()
11099            .pop()
11100            .unwrap();
11101        let ClientControlResponse::CatalogList {
11102            generation,
11103            modules,
11104            ..
11105        } = serde_json::from_slice(&reply.body).unwrap()
11106        else {
11107            panic!("not a catalog.list reply");
11108        };
11109        (
11110            generation,
11111            modules
11112                .into_iter()
11113                .map(|entry| (entry.module_id, entry.ready))
11114                .collect(),
11115        )
11116    }
11117
11118    /// A handler whose HELLO for module "aft" on connection 40 is parked
11119    /// between reserving its registry entry and installing its forwarding
11120    /// endpoint. Returns the handler, the two-party barrier (wait once to know
11121    /// the HELLO is parked, once more to release it), the HELLO's thread, and
11122    /// the module's context and receiver.
11123    fn hello_held_before_forwarding() -> (
11124        ControlHandler,
11125        Arc<std::sync::Barrier>,
11126        std::thread::JoinHandle<Vec<Frame>>,
11127        RouteCtx,
11128        mpsc::Receiver<crate::router::OutboundFrame>,
11129    ) {
11130        let hold = Arc::new(std::sync::Barrier::new(2));
11131        let handler = ControlHandler::with_forwarding(
11132            Arc::new(Registry::default()),
11133            Arc::new(ForwardingTable::default()),
11134        )
11135        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11136        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11137        let hello = {
11138            let handler = handler.clone();
11139            let module_ctx = module_ctx.clone();
11140            std::thread::spawn(move || {
11141                handler
11142                    .handle_hello(
11143                        module_ctx.connection_id,
11144                        Some(module_ctx.egress.clone()),
11145                        hello_frame("aft", PROTOCOL_VERSION, 7),
11146                    )
11147                    .unwrap()
11148            })
11149        };
11150        (handler, hold, hello, module_ctx, module_rx)
11151    }
11152
11153    /// While a HELLO has reserved its registry entry but its forwarding
11154    /// endpoint is not installed yet, `route.open` to the module can only be
11155    /// refused. `catalog.list` must not list the module in that window: a
11156    /// client that opens a route as soon as it sees a module listed and ready
11157    /// would be refused for a module the daemon had just advertised. The
11158    /// refusal in the window stays the retryable `target_unavailable`, never
11159    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
11160    /// Once the HELLO completes, the module is listed and a route to it opens.
11161    #[tokio::test]
11162    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11163        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11164        let client = ConnectionId::new(41);
11165        let (client_ctx, mut client_rx) = route_ctx(client);
11166
11167        hold.wait();
11168        // Parked: the registry holds "aft", forwarding has no endpoint for it.
11169        assert!(handler.registry.get_module("aft").unwrap().is_some());
11170        assert!(!handler
11171            .forwarding
11172            .has_live_module_connection("aft")
11173            .unwrap());
11174        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11175        assert!(
11176            listed.is_empty(),
11177            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11178        );
11179        let refused = handler
11180            .handle_control_frame(
11181                &client_ctx,
11182                route_open_frame(301, "aft", unique_project_root("listed-window")),
11183            )
11184            .await
11185            .unwrap();
11186        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11187        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11188        let duplicate = handler
11189            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11190            .await
11191            .unwrap();
11192        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11193
11194        hold.wait();
11195        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11196        let ack = module_rx.try_recv().unwrap().frame;
11197        assert_eq!(ack.header.ty, FrameType::HelloAck);
11198        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11199        assert_eq!(listed, vec![("aft".to_string(), true)]);
11200
11201        let (open_task, bind) = relay_route_open(
11202            &handler,
11203            client,
11204            &client_ctx.egress,
11205            &mut module_rx,
11206            303,
11207            "aft",
11208            "listed-after",
11209        )
11210        .await;
11211        assert!(handler
11212            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11213            .await
11214            .unwrap()
11215            .is_empty());
11216        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11217        published_route(&client_rx.try_recv().unwrap().frame);
11218    }
11219
11220    /// A catalog watcher keys on the generation: when it changes, it re-reads
11221    /// the listing. So the generation must advance exactly when the module
11222    /// appears. If it advanced while the module was still hidden, a watcher
11223    /// would read the new generation without the module, and the module would
11224    /// later appear under that same generation with nothing telling the
11225    /// watcher to look again.
11226    #[tokio::test]
11227    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11228        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11229        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11230        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11231
11232        hold.wait();
11233        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11234        assert!(listed.is_empty());
11235        assert_eq!(
11236            during, before,
11237            "the generation moved while the module was not listed"
11238        );
11239
11240        hold.wait();
11241        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11242        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11243        assert_eq!(listed, vec![("aft".to_string(), true)]);
11244        assert_eq!(
11245            after,
11246            before + 1,
11247            "the module appeared without exactly one generation bump"
11248        );
11249    }
11250
11251    /// A HELLO whose forwarding endpoint cannot be installed (here the
11252    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11253    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11254    /// and the catalog generation where it was.
11255    #[tokio::test]
11256    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11257        let registry = Arc::new(Registry::default());
11258        let handler = ControlHandler::with_forwarding(
11259            Arc::clone(&registry),
11260            Arc::new(ForwardingTable::default()),
11261        );
11262        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11263        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11264        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11265        drop(module_rx);
11266
11267        let replies = handler
11268            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11269            .await
11270            .unwrap();
11271        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11272        assert!(registry.get_module("aft").unwrap().is_none());
11273        assert!(registry
11274            .get_module_by_connection(module_ctx.connection_id)
11275            .unwrap()
11276            .is_none());
11277        assert_eq!(
11278            catalog_snapshot(&handler, &client_ctx, 301).await,
11279            (before, Vec::new())
11280        );
11281    }
11282
11283    /// A handler whose `route.open` parks after queueing the route.bind and
11284    /// before marking it enqueued, with module "aft" registered on connection
11285    /// 40. Returns the two-party barrier that parks and releases it.
11286    async fn route_open_held_after_bind_send() -> (
11287        ControlHandler,
11288        Arc<tokio::sync::Barrier>,
11289        RouteCtx,
11290        mpsc::Receiver<crate::router::OutboundFrame>,
11291    ) {
11292        let hold = Arc::new(tokio::sync::Barrier::new(2));
11293        let handler = ControlHandler::with_forwarding(
11294            Arc::new(Registry::default()),
11295            Arc::new(ForwardingTable::default()),
11296        )
11297        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11298        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11299        hello_via_sink(
11300            &handler,
11301            &module_ctx,
11302            &mut module_rx,
11303            hello_frame("aft", PROTOCOL_VERSION, 7),
11304        )
11305        .await;
11306        (handler, hold, module_ctx, module_rx)
11307    }
11308
11309    /// A fast module acks the route.bind, and the reader commits the route,
11310    /// before `route.open` has marked the relay enqueued. The daemon used to
11311    /// read the missing relay as an abandoned bind and send the module a
11312    /// GOODBYE for the route it had just committed: the client was told its
11313    /// route was open, the module unbound the channel, and the client's first
11314    /// request was dropped on the module side as unbound.
11315    #[tokio::test]
11316    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11317        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11318        let client = ConnectionId::new(41);
11319        let (client_ctx, mut client_rx) = route_ctx(client);
11320
11321        let (open_task, bind) = relay_route_open(
11322            &handler,
11323            client,
11324            &client_ctx.egress,
11325            &mut module_rx,
11326            200,
11327            "aft",
11328            "acked-before-mark",
11329        )
11330        .await;
11331        let (module_channel, module_epoch) = route_bind_channel(&bind);
11332        // route.open is now parked between queueing the bind and marking it.
11333        hold.wait().await;
11334
11335        // The module answers inside that window, and the route commits.
11336        assert!(handler
11337            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11338            .await
11339            .unwrap()
11340            .is_empty());
11341        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11342
11343        hold.wait().await;
11344        assert!(
11345            open_task.await.unwrap().is_empty(),
11346            "route.open answers accepted"
11347        );
11348
11349        // Every frame route.open could have sent is already queued, because it
11350        // has returned; the module must have been sent nothing.
11351        match module_rx.try_recv() {
11352            Err(mpsc::error::TryRecvError::Empty) => {}
11353            Ok(outbound) => panic!(
11354                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11355                outbound.frame.header.ty,
11356                outbound.frame.header.channel,
11357                outbound.frame.header.epoch
11358            ),
11359            Err(err) => panic!("module egress closed: {err}"),
11360        }
11361
11362        // The client's first request on the route is the next thing the module
11363        // reads, on the binding it just created.
11364        let router = Router::with_control_handler(Arc::new(handler.clone()));
11365        let request = Frame::build(
11366            FrameType::Request,
11367            Flags::new(true, Priority::Interactive, false),
11368            client_channel,
11369            client_epoch,
11370            201,
11371            b"first request".to_vec(),
11372        )
11373        .unwrap();
11374        router
11375            .route_for_connection(&client_ctx, request)
11376            .await
11377            .unwrap();
11378        let delivered = module_rx.try_recv().unwrap().frame;
11379        assert_eq!(delivered.header.ty, FrameType::Request);
11380        assert_eq!(
11381            (delivered.header.channel, delivered.header.epoch),
11382            (module_channel, module_epoch)
11383        );
11384        assert_eq!(delivered.body, b"first request");
11385    }
11386
11387    /// The client goes away while `route.open` is still queueing the bind, so
11388    /// the relay is dropped with no module reply. The teardown cannot send the
11389    /// GOODBYE itself (it could reach the module ahead of the bind), so
11390    /// `route.open` must send it once the bind is queued; otherwise the module
11391    /// keeps a binding the daemon will never commit.
11392    #[tokio::test]
11393    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11394        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11395        let client = ConnectionId::new(42);
11396        let (client_ctx, _client_rx) = route_ctx(client);
11397
11398        let (open_task, bind) = relay_route_open(
11399            &handler,
11400            client,
11401            &client_ctx.egress,
11402            &mut module_rx,
11403            300,
11404            "aft",
11405            "released-before-mark",
11406        )
11407        .await;
11408        let (module_channel, module_epoch) = route_bind_channel(&bind);
11409        hold.wait().await;
11410
11411        handler.cleanup_connection(client).unwrap();
11412        assert!(
11413            module_rx.try_recv().is_err(),
11414            "client teardown must not GOODBYE a bind route.open is still queueing"
11415        );
11416
11417        hold.wait().await;
11418        let response = open_task.await.unwrap();
11419        assert_eq!(response.len(), 1);
11420        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11421
11422        let goodbye = module_rx
11423            .try_recv()
11424            .expect("module is told to drop the binding the daemon abandoned")
11425            .frame;
11426        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11427        assert_eq!(
11428            (goodbye.header.channel, goodbye.header.epoch),
11429            (module_channel, module_epoch)
11430        );
11431        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11432    }
11433
11434    /// The module acks inside the same window, but the client's egress closed
11435    /// first, so the reader refuses to commit. The ack proves the module holds
11436    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11437    /// send a second one.
11438    #[tokio::test]
11439    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11440        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11441        let client = ConnectionId::new(43);
11442        let (client_ctx, client_rx) = route_ctx(client);
11443
11444        let (open_task, bind) = relay_route_open(
11445            &handler,
11446            client,
11447            &client_ctx.egress,
11448            &mut module_rx,
11449            400,
11450            "aft",
11451            "gone-client-before-mark",
11452        )
11453        .await;
11454        let (module_channel, module_epoch) = route_bind_channel(&bind);
11455        hold.wait().await;
11456
11457        drop(client_rx);
11458        assert!(client_ctx.egress.is_closed());
11459        handler
11460            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11461            .await
11462            .unwrap();
11463
11464        hold.wait().await;
11465        let response = open_task.await.unwrap();
11466        assert_eq!(response.len(), 1);
11467        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11468
11469        let goodbye = module_rx
11470            .try_recv()
11471            .expect("module is told to drop the binding it acked")
11472            .frame;
11473        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11474        assert_eq!(
11475            (goodbye.header.channel, goodbye.header.epoch),
11476            (module_channel, module_epoch)
11477        );
11478        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11479        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11480    }
11481
11482    /// The fence at the module-loop boundary, stated as its own contract: which
11483    /// forwarding failures are allowed to end the module connection that is being
11484    /// served. A `ConnectionClosing` naming some client is about that client, and
11485    /// a module connection is shared; the same error naming the module's own
11486    /// connection is about this connection and must stay fatal, as must failures
11487    /// that are about the forwarding table itself.
11488    #[test]
11489    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11490        let handler = ControlHandler::default();
11491        let module_connection = ConnectionId::new(30);
11492        let client_connection = ConnectionId::new(31);
11493
11494        handler
11495            .refuse_to_end_module_connection_for_a_client(
11496                module_connection,
11497                77,
11498                ForwardingError::ConnectionClosing {
11499                    connection_id: client_connection,
11500                },
11501            )
11502            .expect("a closing client must never end the module connection");
11503
11504        assert!(matches!(
11505            handler.refuse_to_end_module_connection_for_a_client(
11506                module_connection,
11507                78,
11508                ForwardingError::ConnectionClosing {
11509                    connection_id: module_connection,
11510                },
11511            ),
11512            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11513                connection_id
11514            })) if connection_id == module_connection
11515        ));
11516        assert!(matches!(
11517            handler.refuse_to_end_module_connection_for_a_client(
11518                module_connection,
11519                79,
11520                ForwardingError::Poisoned,
11521            ),
11522            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11523        ));
11524        assert!(matches!(
11525            handler.refuse_to_end_module_connection_for_a_client(
11526                module_connection,
11527                80,
11528                ForwardingError::StaleModuleEndpoint,
11529            ),
11530            Err(RouterError::Forwarding(
11531                ForwardingError::StaleModuleEndpoint
11532            ))
11533        ));
11534    }
11535
11536    /// The spawn-attestation guard is what stops a connected module from claiming
11537    /// another module's identity and being stamped `Reserved` for it. Every other
11538    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11539    /// correct one is what the rest of the flow needs -- so the guard's rejection
11540    /// branch was never the subject of an assertion, only its acceptance branch.
11541    ///
11542    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11543    /// whole subc-core library suite green; only the forwarding integration tests
11544    /// notice, and they notice for unrelated reasons. This test exists so the
11545    /// refusal itself is asserted where the guard lives: it fails if the identity
11546    /// check stops refusing, which is the direction that matters, since a guard
11547    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11548    #[tokio::test]
11549    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11550        let registry = Arc::new(Registry::default());
11551        let forwarding = Arc::new(ForwardingTable::default());
11552        let supervisor = SupervisorHandle::new();
11553        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11554        let handler =
11555            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11556                .with_supervisor(supervisor);
11557
11558        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11559        hello_via_sink(
11560            &handler,
11561            &target_ctx,
11562            &mut target_rx,
11563            hello_frame("target", PROTOCOL_VERSION, 1),
11564        )
11565        .await;
11566
11567        // A real supervised module id presenting the wrong nonce. This is the
11568        // impersonation case: the attacker knows a privileged module_id, which is
11569        // public, and guesses at the nonce, which is not.
11570        let wrong_nonce = handler
11571            .handle_control_frame(
11572                &route_ctx(ConnectionId::new(91)).0,
11573                route_open_frame_with_admission_facts(
11574                    20,
11575                    "target",
11576                    unique_project_root("admission-facts"),
11577                    Some(subc_control::ConsumerIdentity {
11578                        module_id: "fed".to_string(),
11579                        launch_nonce: "not-the-real-nonce".to_string(),
11580                    }),
11581                    None,
11582                ),
11583            )
11584            .await
11585            .unwrap();
11586        assert_eq!(
11587            parse_error(&wrong_nonce[0])["code"],
11588            "bad_consumer_identity",
11589            "a mismatched launch nonce must be refused, not stamped Reserved"
11590        );
11591
11592        // A module id the supervisor never spawned at all, so no nonce exists to
11593        // compare against. An implementation that treats "no record" as "nothing
11594        // to check" fails open here while passing the case above.
11595        let never_spawned = handler
11596            .handle_control_frame(
11597                &route_ctx(ConnectionId::new(92)).0,
11598                route_open_frame_with_admission_facts(
11599                    21,
11600                    "target",
11601                    unique_project_root("admission-facts"),
11602                    Some(subc_control::ConsumerIdentity {
11603                        module_id: "never-spawned".to_string(),
11604                        launch_nonce: "any-nonce".to_string(),
11605                    }),
11606                    None,
11607                ),
11608            )
11609            .await
11610            .unwrap();
11611        assert_eq!(
11612            parse_error(&never_spawned[0])["code"],
11613            "bad_consumer_identity",
11614            "an unspawned module_id must be refused rather than accepted for lack of a record"
11615        );
11616    }
11617
11618    /// The refusal test above proves the guard says NO. Nothing proved it can say
11619    /// YES, and the difference is not academic: replacing the whole authorization
11620    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11621    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11622    /// library tests GREEN. The one that notices does so by HANGING, because it
11623    /// waits for a bind that can no longer happen.
11624    ///
11625    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11626    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11627    /// hangs too and gets blamed on the runner. So a total revocation of the
11628    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11629    /// to code.
11630    ///
11631    /// The bias is structural rather than accidental. A REFUSAL looks like a
11632    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11633    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11634    /// suite for that reason, and this one is the purest case in the daemon.
11635    ///
11636    /// This test asserts the EFFECT rather than the absence of an error: the module
11637    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11638    /// A guard that admitted nobody would produce no bind at all; one that admitted
11639    /// everybody would stamp the wrong principal, which the refusal test catches.
11640    #[tokio::test]
11641    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11642        let registry = Arc::new(Registry::default());
11643        let forwarding = Arc::new(ForwardingTable::default());
11644        let supervisor = SupervisorHandle::new();
11645        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11646        let handler =
11647            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11648                .with_supervisor(supervisor);
11649
11650        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11651        hello_via_sink(
11652            &handler,
11653            &target_ctx,
11654            &mut target_rx,
11655            hello_frame("target", PROTOCOL_VERSION, 1),
11656        )
11657        .await;
11658
11659        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11660        let route_handler = handler.clone();
11661        let route_task = tokio::spawn(async move {
11662            route_handler
11663                .handle_control_frame(
11664                    &client_ctx,
11665                    route_open_frame_with_admission_facts(
11666                        30,
11667                        "target",
11668                        unique_project_root("admission-facts"),
11669                        Some(subc_control::ConsumerIdentity {
11670                            module_id: "fed".to_string(),
11671                            launch_nonce: "fed-nonce".to_string(),
11672                        }),
11673                        None,
11674                    ),
11675                )
11676                .await
11677                .unwrap()
11678        });
11679
11680        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11681        // the very mutation it exists to catch -- a guard that admits nobody -- no
11682        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11683        // exact defect being fixed: a total revocation detected only as a stalled
11684        // suite, which reads as flakiness and invites a retry. An acceptance test
11685        // that waits for an effect must bound the wait, or a red becomes a hang.
11686        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11687            .await
11688            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11689            .expect("module control channel closed before route.bind");
11690        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11691        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11692            panic!("expected route.bind")
11693        };
11694        assert_eq!(
11695            principal,
11696            Some(Principal::Reserved {
11697                module_id: "fed".to_string()
11698            }),
11699            "a correctly attested consumer must be stamped Reserved for its own id"
11700        );
11701
11702        handler
11703            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11704            .await
11705            .unwrap();
11706        assert!(route_task.await.unwrap().is_empty());
11707        assert!(
11708            matches!(
11709                serde_json::from_slice::<ClientControlResponse>(
11710                    &client_rx.recv().await.unwrap().body
11711                )
11712                .unwrap(),
11713                ClientControlResponse::RouteOpen { .. }
11714            ),
11715            "the route must actually open, not merely avoid an error"
11716        );
11717    }
11718
11719    #[tokio::test(start_paused = true)]
11720    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11721        let registry = Arc::new(Registry::default());
11722        let forwarding = Arc::new(ForwardingTable::default());
11723        let supervisor = SupervisorHandle::new();
11724        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11725        let handler =
11726            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11727                .with_supervisor(supervisor);
11728
11729        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11730        hello_via_sink(
11731            &handler,
11732            &target_ctx,
11733            &mut target_rx,
11734            hello_frame("target", PROTOCOL_VERSION, 1),
11735        )
11736        .await;
11737
11738        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11739        let direct_handler = handler.clone();
11740        let direct_open = tokio::spawn(async move {
11741            direct_handler
11742                .handle_control_frame(
11743                    &direct_ctx,
11744                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11745                )
11746                .await
11747                .unwrap()
11748        });
11749        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11750            .await
11751            .expect("no direct route.bind within 5s")
11752            .expect("target control channel closed before direct route.bind");
11753        handler
11754            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11755            .await
11756            .unwrap();
11757        assert!(direct_open.await.unwrap().is_empty());
11758        let _ = direct_rx.recv().await.unwrap();
11759
11760        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11761        let reserved_handler = handler.clone();
11762        let reserved_open = tokio::spawn(async move {
11763            reserved_handler
11764                .handle_control_frame(
11765                    &reserved_ctx,
11766                    route_open_frame_with_admission_facts(
11767                        3,
11768                        "target",
11769                        unique_project_root("admission-facts"),
11770                        Some(ConsumerIdentity {
11771                            module_id: "fed".to_string(),
11772                            launch_nonce: "fed-nonce".to_string(),
11773                        }),
11774                        None,
11775                    ),
11776                )
11777                .await
11778                .unwrap()
11779        });
11780        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11781            .await
11782            .expect("no reserved route.bind within 5s")
11783            .expect("target control channel closed before reserved route.bind");
11784        handler
11785            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11786            .await
11787            .unwrap();
11788        assert!(reserved_open.await.unwrap().is_empty());
11789        let _ = reserved_rx.recv().await.unwrap();
11790
11791        forwarding
11792            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11793            .unwrap();
11794        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11795        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11796            module_id: Some("target".to_string()),
11797        })
11798        .unwrap();
11799        let census_frame =
11800            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11801        let response = handler
11802            .handle_control_frame(&census_ctx, census_frame)
11803            .await
11804            .unwrap()
11805            .pop()
11806            .unwrap();
11807        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11808        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11809        assert!(matches!(
11810            decoded,
11811            ClientControlResponse::SupervisorRoutes { .. }
11812        ));
11813        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11814        assert_eq!(routes.len(), 2);
11815        assert!(routes.iter().all(|route| route["draining"] == true));
11816        // The census carries WHY: the reason the drain was begun with, in the
11817        // route.closing vocabulary, on every draining route this drain marked.
11818        assert!(
11819            routes.iter().all(|route| route["drain_reason"] == "reload"),
11820            "draining routes must name the drain's reason: {routes:?}"
11821        );
11822        assert!(routes.iter().any(|route| {
11823            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11824        }));
11825        assert!(routes.iter().any(|route| {
11826            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11827        }));
11828
11829        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11830            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11831        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11832            std::fs::write(
11833                &golden_path,
11834                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11835            )
11836            .unwrap();
11837        }
11838        let expected: Value =
11839            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11840        assert_eq!(actual, expected);
11841    }
11842
11843    async fn query_live_roots(
11844        handler: &ControlHandler,
11845        module_ctx: &RouteCtx,
11846    ) -> ModuleControlResponseToModule {
11847        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11848        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11849        let response = handler
11850            .handle_control_frame(module_ctx, frame)
11851            .await
11852            .unwrap()
11853            .pop()
11854            .unwrap();
11855        serde_json::from_slice(&response.body).unwrap()
11856    }
11857
11858    #[tokio::test(start_paused = true)]
11859    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11860        let registry = Arc::new(Registry::default());
11861        let forwarding = Arc::new(ForwardingTable::default());
11862        let handler = ControlHandler::with_forwarding(registry, forwarding);
11863        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11864        hello_via_sink(
11865            &handler,
11866            &target_ctx,
11867            &mut target_rx,
11868            hello_frame("target", PROTOCOL_VERSION, 1),
11869        )
11870        .await;
11871        let root = unique_project_root("live-roots-known");
11872        let path = ProjectRootId::from_path_allowing_missing(root.path())
11873            .unwrap()
11874            .as_path()
11875            .to_path_buf();
11876        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11877        let open_handler = handler.clone();
11878        let opened = tokio::spawn(async move {
11879            open_handler
11880                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11881                .await
11882                .unwrap()
11883        });
11884        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11885            .await
11886            .unwrap()
11887            .unwrap();
11888        handler
11889            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11890            .await
11891            .unwrap();
11892        assert!(opened.await.unwrap().is_empty());
11893        let _ = client_rx.recv().await.unwrap();
11894
11895        let root = unique_project_root("live-roots-pending");
11896        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11897            .unwrap()
11898            .as_path()
11899            .to_path_buf();
11900        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11901        let open_handler = handler.clone();
11902        let pending = tokio::spawn(async move {
11903            open_handler
11904                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11905                .await
11906                .unwrap()
11907        });
11908        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11909            .await
11910            .unwrap()
11911            .unwrap();
11912        let actual = query_live_roots(&handler, &target_ctx).await;
11913        let ModuleControlResponseToModule::LiveRoots {
11914            roots,
11915            unknown_root_bindings,
11916            total_bindings,
11917        } = actual
11918        else {
11919            panic!("expected live roots")
11920        };
11921        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11922        assert_eq!(unknown_root_bindings, 0);
11923        assert_eq!(
11924            roots.len(),
11925            2,
11926            "root-known arm must retain each canonical root"
11927        );
11928        assert_eq!(
11929            total_bindings,
11930            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11931        );
11932        let counts = roots
11933            .iter()
11934            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11935            .collect::<Vec<_>>();
11936        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11937        expected.sort_by(|a, b| a.0.cmp(&b.0));
11938        assert_eq!(
11939            counts, expected,
11940            "roots must sort by path and count pending separately"
11941        );
11942        handler
11943            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11944            .await
11945            .unwrap();
11946        assert!(pending.await.unwrap().is_empty());
11947    }
11948
11949    #[tokio::test(start_paused = true)]
11950    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11951        let forwarding = Arc::new(ForwardingTable::default());
11952        let handler =
11953            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11954        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11955        hello_via_sink(
11956            &handler,
11957            &target_ctx,
11958            &mut target_rx,
11959            hello_frame("target", PROTOCOL_VERSION, 1),
11960        )
11961        .await;
11962        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11963        let pending = forwarding
11964            .begin_route_bind_relay_for_test(
11965                client_ctx.connection_id,
11966                client_ctx.egress.clone(),
11967                2,
11968                "target",
11969            )
11970            .unwrap();
11971        forwarding
11972            .complete_pending_relay(
11973                target_ctx.connection_id,
11974                pending.corr,
11975                RouteBindRelayOutcome::Accepted,
11976            )
11977            .unwrap();
11978        let actual = query_live_roots(&handler, &target_ctx).await;
11979        let ModuleControlResponseToModule::LiveRoots {
11980            roots,
11981            unknown_root_bindings,
11982            total_bindings,
11983        } = actual
11984        else {
11985            panic!("expected live roots")
11986        };
11987        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11988        assert_eq!(
11989            unknown_root_bindings, 1,
11990            "unknown-root arm must not read as no bindings"
11991        );
11992        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11993        assert_eq!(
11994            total_bindings,
11995            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11996        );
11997    }
11998
11999    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12000    /// so the ack has to be on its outbound queue before the module is
12001    /// routable. The connection loop writes a handler's replies only after the
12002    /// handler returns; this test stops in exactly that gap, runs a real
12003    /// route.open from another connection, and only then writes whatever the
12004    /// HELLO handler returned, the way the loop would. If the ack were still a
12005    /// reply, the route.bind request would reach the module first.
12006    #[tokio::test(start_paused = true)]
12007    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12008        let forwarding = Arc::new(ForwardingTable::default());
12009        let handler =
12010            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12011        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12012        let replies = handler
12013            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12014            .await
12015            .unwrap();
12016        let queued_by_hello = module_rx.len();
12017
12018        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12019        let open_handler = handler.clone();
12020        let open = tokio::spawn(async move {
12021            open_handler
12022                .handle_control_frame(
12023                    &client_ctx,
12024                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12025                )
12026                .await
12027                .unwrap()
12028        });
12029        // Let the route.open run until its route.bind is on the module's queue.
12030        let mut spins = 0;
12031        while module_rx.len() == queued_by_hello {
12032            spins += 1;
12033            assert!(spins < 10_000, "route.open never queued a route.bind");
12034            tokio::task::yield_now().await;
12035        }
12036
12037        // Now the connection loop's half: write the HELLO handler's replies.
12038        for reply in replies {
12039            module_ctx.egress.send(reply).await.unwrap();
12040        }
12041
12042        let first = module_rx.recv().await.unwrap().frame;
12043        assert_eq!(
12044            first.header.ty,
12045            FrameType::HelloAck,
12046            "the first frame a registering module reads must be its HELLO_ACK"
12047        );
12048        assert_eq!(first.header.corr, 7);
12049        let second = module_rx.recv().await.unwrap().frame;
12050        assert_eq!(second.header.ty, FrameType::Request);
12051        assert!(
12052            matches!(
12053                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12054                ModuleControlRequest::RouteBind { .. }
12055            ),
12056            "the route.bind follows the ack"
12057        );
12058        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12059
12060        handler
12061            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12062            .await
12063            .unwrap();
12064        assert!(open.await.unwrap().is_empty());
12065        let _ = client_rx.recv().await.unwrap();
12066    }
12067
12068    #[tokio::test(start_paused = true)]
12069    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12070        let handler = ControlHandler::with_forwarding(
12071            Arc::new(Registry::default()),
12072            Arc::new(ForwardingTable::default()),
12073        );
12074        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12075        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12076        hello_via_sink(
12077            &handler,
12078            &first_ctx,
12079            &mut first_rx,
12080            hello_frame("first", PROTOCOL_VERSION, 1),
12081        )
12082        .await;
12083        hello_via_sink(
12084            &handler,
12085            &second_ctx,
12086            &mut second_rx,
12087            hello_frame("second", PROTOCOL_VERSION, 2),
12088        )
12089        .await;
12090        let root = unique_project_root("second-only");
12091        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12092        let cloned = handler.clone();
12093        let open = tokio::spawn(async move {
12094            cloned
12095                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12096                .await
12097                .unwrap()
12098        });
12099        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12100            .await
12101            .unwrap()
12102            .unwrap();
12103        let first = query_live_roots(&handler, &first_ctx).await;
12104        let second = query_live_roots(&handler, &second_ctx).await;
12105        assert!(
12106            matches!(
12107                first,
12108                ModuleControlResponseToModule::LiveRoots {
12109                    total_bindings: 0,
12110                    ..
12111                }
12112            ),
12113            "cross-module scope must not expose another module's roots"
12114        );
12115        assert!(
12116            matches!(
12117                second,
12118                ModuleControlResponseToModule::LiveRoots {
12119                    total_bindings: 1,
12120                    ..
12121                }
12122            ),
12123            "second module must see its pending route"
12124        );
12125        handler
12126            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12127            .await
12128            .unwrap();
12129        assert!(open.await.unwrap().is_empty());
12130    }
12131
12132    #[tokio::test(start_paused = true)]
12133    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12134        let handler = ControlHandler::with_forwarding(
12135            Arc::new(Registry::default()),
12136            Arc::new(ForwardingTable::default()),
12137        );
12138        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12139        hello_via_sink(
12140            &handler,
12141            &target_ctx,
12142            &mut target_rx,
12143            hello_frame("target", PROTOCOL_VERSION, 1),
12144        )
12145        .await;
12146        let actual = query_live_roots(&handler, &target_ctx).await;
12147        let ModuleControlResponseToModule::LiveRoots {
12148            roots,
12149            unknown_root_bindings,
12150            total_bindings,
12151        } = actual
12152        else {
12153            panic!("expected live roots")
12154        };
12155        assert!(roots.is_empty());
12156        assert_eq!(unknown_root_bindings, 0);
12157        assert_eq!(total_bindings, 0);
12158        assert_eq!(
12159            total_bindings,
12160            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12161        );
12162    }
12163
12164    /// Read the vendored fed corpus rather than hand-building a package.
12165    ///
12166    /// A hand-built object encodes what the test author believed the carrier
12167    /// emits. These vectors are what it actually emits, and one of them exists
12168    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
12169    /// ignores additive unknown fields at the traversal emit terminus."
12170    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12171        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12172            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12173        let text = std::fs::read_to_string(&path)
12174            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12175        let vectors: Vec<(String, Value)> = text
12176            .lines()
12177            .filter(|line| !line.trim().is_empty())
12178            .map(|line| {
12179                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12180                let id = entry["corpus_id"]
12181                    .as_str()
12182                    .expect("every vector carries a corpus_id")
12183                    .to_string();
12184                (id, entry["package"].clone())
12185            })
12186            .collect();
12187        // Pin the count: a corpus that silently shrinks would take its coverage
12188        // with it, and a suite reading N-1 vectors reports the same clean pass
12189        // as one reading N.
12190        assert_eq!(
12191            vectors.len(),
12192            3,
12193            "vendored fed corpus changed size; re-sync from subc-federation"
12194        );
12195
12196        // Pin what makes the corpus DISCRIMINATING, not just present.
12197        //
12198        // The relay test below takes its expected value from the corpus, so the
12199        // corpus supplies the test's power to detect a lossy relay rather than
12200        // its correctness. A relay that dropped unrecognised fields would still
12201        // be caught -- but only by a package carrying fields it does not know.
12202        // Shrink every package to the handful of keys any implementation would
12203        // recognise and the test keeps passing over an input that can no longer
12204        // fail, which is the same clean green as a corpus that shrank away.
12205        //
12206        // So assert the precondition rather than duplicating the packages here:
12207        // at least one vector must carry a field beyond the small common set.
12208        // That is one claim to maintain instead of nine, and it fails loudly if
12209        // a re-sync ever flattens the corpus.
12210        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12211        let richest = vectors
12212            .iter()
12213            .filter_map(|(_, package)| package.as_object())
12214            .map(|object| {
12215                object
12216                    .keys()
12217                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12218                    .count()
12219            })
12220            .max()
12221            .unwrap_or(0);
12222        assert!(
12223            richest >= 2,
12224            "vendored corpus no longer carries a package with unmodelled fields, \
12225             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12226        );
12227
12228        vectors
12229    }
12230
12231    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12232    ///
12233    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12234    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12235    /// three-key object, so a relay that quietly dropped fields it did not
12236    /// recognise would satisfy it. These vectors carry nine keys including ones
12237    /// this crate has no type for, so a typed relay fails here and only here.
12238    #[tokio::test]
12239    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12240        for (corpus_id, package) in fed_admission_facts_vectors() {
12241            let registry = Arc::new(Registry::default());
12242            let forwarding = Arc::new(ForwardingTable::default());
12243            let supervisor = SupervisorHandle::new();
12244            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12245            let handler =
12246                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12247                    .with_supervisor(supervisor)
12248                    .with_admission_facts_config(
12249                        Some("fed".to_string()),
12250                        Some(vec!["target".to_string()]),
12251                    );
12252
12253            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12254            hello_via_sink(
12255                &handler,
12256                &target_ctx,
12257                &mut target_rx,
12258                hello_frame("target", PROTOCOL_VERSION, 1),
12259            )
12260            .await;
12261
12262            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12263            let route_handler = handler.clone();
12264            let expected = package.clone();
12265            let route_task = tokio::spawn(async move {
12266                route_handler
12267                    .handle_control_frame(
12268                        &client_ctx,
12269                        route_open_frame_with_admission_facts(
12270                            20,
12271                            "target",
12272                            unique_project_root("admission-facts"),
12273                            Some(subc_control::ConsumerIdentity {
12274                                module_id: "fed".to_string(),
12275                                launch_nonce: "fed-nonce".to_string(),
12276                            }),
12277                            Some(package),
12278                        ),
12279                    )
12280                    .await
12281                    .unwrap()
12282            });
12283
12284            let bind_frame = target_rx.recv().await.unwrap();
12285            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12286            let ModuleControlRequest::RouteBind {
12287                admission_facts, ..
12288            } = bind
12289            else {
12290                panic!("{corpus_id}: expected route.bind")
12291            };
12292            assert_eq!(
12293                admission_facts,
12294                Some(expected),
12295                "{corpus_id}: relay must not add, drop or reshape any field"
12296            );
12297
12298            handler
12299                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12300                .await
12301                .unwrap();
12302            route_task.await.unwrap();
12303        }
12304    }
12305
12306    #[tokio::test]
12307    async fn route_bind_never_relays_the_openers_build_class() {
12308        let registry = Arc::new(Registry::default());
12309        let forwarding = Arc::new(ForwardingTable::default());
12310        let handler =
12311            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12312        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12313        hello_via_sink(
12314            &handler,
12315            &target_ctx,
12316            &mut target_rx,
12317            hello_frame("target", PROTOCOL_VERSION, 1),
12318        )
12319        .await;
12320
12321        let root = unique_project_root("build-class");
12322        let mut identity = BindIdentity::new(
12323            root.path().to_path_buf(),
12324            "unit".to_string(),
12325            "s".to_string(),
12326        );
12327        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12328        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12329            target: RouteTarget::ToolProvider {
12330                module_id: "target".to_string(),
12331            },
12332            identity,
12333            consumer_identity: None,
12334            consumer_capabilities: None,
12335            role_versions: None,
12336            admission_facts: None,
12337            scope: None,
12338        })
12339        .unwrap();
12340        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12341
12342        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12343        let route_handler = handler.clone();
12344        let route_task = tokio::spawn(async move {
12345            route_handler
12346                .handle_control_frame(&client_ctx, open)
12347                .await
12348                .unwrap()
12349        });
12350        let bind_frame = target_rx.recv().await.unwrap();
12351        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12352        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12353            panic!("expected route.bind")
12354        };
12355        assert_eq!(
12356            identity.build_class, None,
12357            "route.bind carried the build class"
12358        );
12359        assert!(
12360            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12361            "the route.bind bytes still name build_class"
12362        );
12363        handler
12364            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12365            .await
12366            .unwrap();
12367        route_task.await.unwrap();
12368    }
12369
12370    #[tokio::test]
12371    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12372        let registry = Arc::new(Registry::default());
12373        let forwarding = Arc::new(ForwardingTable::default());
12374        let supervisor = SupervisorHandle::new();
12375        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12376        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12377        let handler =
12378            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12379                .with_supervisor(supervisor)
12380                .with_admission_facts_config(
12381                    Some("fed".to_string()),
12382                    Some(vec!["target".to_string()]),
12383                );
12384
12385        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12386        hello_via_sink(
12387            &handler,
12388            &target_ctx,
12389            &mut target_rx,
12390            hello_frame("target", PROTOCOL_VERSION, 1),
12391        )
12392        .await;
12393        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12394        hello_via_sink(
12395            &handler,
12396            &other_ctx,
12397            &mut other_rx,
12398            hello_frame("other", PROTOCOL_VERSION, 2),
12399        )
12400        .await;
12401
12402        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12403        let expected_facts = facts.clone();
12404        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12405        let route_handler = handler.clone();
12406        let route_task = tokio::spawn(async move {
12407            route_handler
12408                .handle_control_frame(
12409                    &client_ctx,
12410                    route_open_frame_with_admission_facts(
12411                        10,
12412                        "target",
12413                        unique_project_root("admission-facts"),
12414                        Some(subc_control::ConsumerIdentity {
12415                            module_id: "fed".to_string(),
12416                            launch_nonce: "fed-nonce".to_string(),
12417                        }),
12418                        Some(facts.clone()),
12419                    ),
12420                )
12421                .await
12422                .unwrap()
12423        });
12424        let bind_frame = target_rx.recv().await.unwrap();
12425        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12426        let ModuleControlRequest::RouteBind {
12427            admission_facts, ..
12428        } = bind
12429        else {
12430            panic!("expected route.bind")
12431        };
12432        assert_eq!(admission_facts, Some(expected_facts));
12433        handler
12434            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12435            .await
12436            .unwrap();
12437        assert!(route_task.await.unwrap().is_empty());
12438        assert!(matches!(
12439            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12440                .unwrap(),
12441            ClientControlResponse::RouteOpen { .. }
12442        ));
12443
12444        let direct = handler
12445            .handle_control_frame(
12446                &route_ctx(ConnectionId::new(73)).0,
12447                route_open_frame_with_admission_facts(
12448                    11,
12449                    "target",
12450                    unique_project_root("admission-facts"),
12451                    None,
12452                    Some(json!({"x": 1})),
12453                ),
12454            )
12455            .await
12456            .unwrap();
12457        assert_eq!(
12458            parse_error(&direct[0])["code"],
12459            "admission_facts_not_permitted"
12460        );
12461
12462        let different_reserved = handler
12463            .handle_control_frame(
12464                &route_ctx(ConnectionId::new(77)).0,
12465                route_open_frame_with_admission_facts(
12466                    15,
12467                    "target",
12468                    unique_project_root("admission-facts"),
12469                    Some(subc_control::ConsumerIdentity {
12470                        module_id: "other".to_string(),
12471                        launch_nonce: "other-nonce".to_string(),
12472                    }),
12473                    Some(json!({"x": 1})),
12474                ),
12475            )
12476            .await
12477            .unwrap();
12478        assert_eq!(
12479            parse_error(&different_reserved[0])["code"],
12480            "admission_facts_not_permitted"
12481        );
12482
12483        let other_target = handler
12484            .handle_control_frame(
12485                &route_ctx(ConnectionId::new(74)).0,
12486                route_open_frame_with_admission_facts(
12487                    12,
12488                    "other",
12489                    unique_project_root("admission-facts"),
12490                    Some(subc_control::ConsumerIdentity {
12491                        module_id: "fed".to_string(),
12492                        launch_nonce: "fed-nonce".to_string(),
12493                    }),
12494                    Some(json!({"x": 1})),
12495                ),
12496            )
12497            .await
12498            .unwrap();
12499        assert_eq!(
12500            parse_error(&other_target[0])["code"],
12501            "admission_facts_target_not_allowed"
12502        );
12503
12504        let nonexistent = handler
12505            .handle_control_frame(
12506                &route_ctx(ConnectionId::new(75)).0,
12507                route_open_frame_with_admission_facts(
12508                    13,
12509                    "missing",
12510                    unique_project_root("admission-facts"),
12511                    None,
12512                    Some(json!({"x": 1})),
12513                ),
12514            )
12515            .await
12516            .unwrap();
12517        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12518
12519        let described = handler
12520            .handle_control_frame(
12521                &route_ctx(ConnectionId::new(76)).0,
12522                Frame::build(
12523                    FrameType::Request,
12524                    control_flags(),
12525                    0,
12526                    0,
12527                    14,
12528                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12529                )
12530                .unwrap(),
12531            )
12532            .await
12533            .unwrap();
12534        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12535            serde_json::from_slice(&described[0].body).unwrap()
12536        else {
12537            panic!("expected server.describe response")
12538        };
12539        assert!(capabilities
12540            .iter()
12541            .any(|cap| cap == "admission_facts_relay_v1"));
12542    }
12543
12544    #[tokio::test]
12545    async fn admission_facts_without_configured_carrier_are_rejected() {
12546        let registry = Arc::new(Registry::default());
12547        let forwarding = Arc::new(ForwardingTable::default());
12548        let handler = ControlHandler::with_forwarding(registry, forwarding);
12549        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12550        hello_via_sink(
12551            &handler,
12552            &target_ctx,
12553            &mut target_rx,
12554            hello_frame("target", PROTOCOL_VERSION, 1),
12555        )
12556        .await;
12557
12558        let responses = handler
12559            .handle_control_frame(
12560                &route_ctx(ConnectionId::new(79)).0,
12561                route_open_frame_with_admission_facts(
12562                    16,
12563                    "target",
12564                    unique_project_root("admission-facts"),
12565                    None,
12566                    Some(json!({"x": 1})),
12567                ),
12568            )
12569            .await
12570            .unwrap();
12571        assert_eq!(
12572            parse_error(&responses[0])["code"],
12573            "admission_facts_not_permitted"
12574        );
12575    }
12576
12577    #[tokio::test]
12578    async fn route_open_relays_consumer_capabilities_verbatim() {
12579        let registry = Arc::new(Registry::default());
12580        let forwarding = Arc::new(ForwardingTable::default());
12581        let handler =
12582            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12583        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12584        hello_via_sink(
12585            &handler,
12586            &module_ctx,
12587            &mut module_rx,
12588            hello_frame("aft", PROTOCOL_VERSION, 7),
12589        )
12590        .await;
12591
12592        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12593        let expected_for_request = expected.clone();
12594        let project_root = unique_project_root("consumer-capabilities-present");
12595        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12596        let route_handler = handler.clone();
12597        let route_task = tokio::spawn(async move {
12598            route_handler
12599                .handle_control_frame(
12600                    &client_ctx,
12601                    route_open_frame_with_consumer_capabilities(
12602                        401,
12603                        "aft",
12604                        project_root,
12605                        Some(expected_for_request),
12606                    ),
12607                )
12608                .await
12609                .unwrap()
12610        });
12611        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12612            .await
12613            .unwrap()
12614            .unwrap();
12615        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12616        let ModuleControlRequest::RouteBind {
12617            consumer_capabilities,
12618            ..
12619        } = bind
12620        else {
12621            panic!("expected route.bind request, got {bind:?}");
12622        };
12623        assert_eq!(consumer_capabilities, Some(expected.clone()));
12624
12625        handler
12626            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12627            .await
12628            .unwrap();
12629        let route_response = route_task.await.unwrap();
12630        assert!(route_response.is_empty());
12631        let published = client_rx.recv().await.unwrap();
12632        assert!(matches!(
12633            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12634            ClientControlResponse::RouteOpen { .. }
12635        ));
12636    }
12637
12638    #[tokio::test]
12639    async fn route_open_without_consumer_capabilities_relays_none() {
12640        let registry = Arc::new(Registry::default());
12641        let forwarding = Arc::new(ForwardingTable::default());
12642        let handler =
12643            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12644        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12645        hello_via_sink(
12646            &handler,
12647            &module_ctx,
12648            &mut module_rx,
12649            hello_frame("aft", PROTOCOL_VERSION, 7),
12650        )
12651        .await;
12652
12653        let project_root = unique_project_root("consumer-capabilities-absent");
12654        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12655        let route_handler = handler.clone();
12656        let route_task = tokio::spawn(async move {
12657            route_handler
12658                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12659                .await
12660                .unwrap()
12661        });
12662        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12663            .await
12664            .unwrap()
12665            .unwrap();
12666        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12667        let ModuleControlRequest::RouteBind {
12668            consumer_capabilities,
12669            ..
12670        } = bind
12671        else {
12672            panic!("expected route.bind request, got {bind:?}");
12673        };
12674        assert_eq!(consumer_capabilities, None);
12675
12676        handler
12677            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12678            .await
12679            .unwrap();
12680        let route_response = route_task.await.unwrap();
12681        assert!(route_response.is_empty());
12682        let published = client_rx.recv().await.unwrap();
12683        assert!(matches!(
12684            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12685            ClientControlResponse::RouteOpen { .. }
12686        ));
12687    }
12688
12689    /// Opens a route to a freshly registered `aft` with `sent` as the
12690    /// route.open's role_versions, acks the bind, and returns the role_versions
12691    /// the module's bind carried.
12692    async fn bind_role_versions_for(
12693        sent: Option<BTreeMap<String, String>>,
12694        connection: u64,
12695    ) -> Option<BTreeMap<String, String>> {
12696        let registry = Arc::new(Registry::default());
12697        let forwarding = Arc::new(ForwardingTable::default());
12698        let handler =
12699            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12700        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12701        hello_via_sink(
12702            &handler,
12703            &module_ctx,
12704            &mut module_rx,
12705            hello_frame("aft", PROTOCOL_VERSION, 7),
12706        )
12707        .await;
12708        let project_root = unique_project_root("role-versions");
12709        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12710        let route_handler = handler.clone();
12711        let route_task = tokio::spawn(async move {
12712            route_handler
12713                .handle_control_frame(
12714                    &client_ctx,
12715                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12716                )
12717                .await
12718                .unwrap()
12719        });
12720        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12721            .await
12722            .expect("a well-formed route.open reaches the module as a bind")
12723            .unwrap();
12724        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12725        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12726            panic!("expected route.bind request, got {bind:?}");
12727        };
12728        handler
12729            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12730            .await
12731            .unwrap();
12732        assert!(route_task.await.unwrap().is_empty());
12733        let published = client_rx.recv().await.unwrap();
12734        assert!(matches!(
12735            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12736            ClientControlResponse::RouteOpen { .. }
12737        ));
12738        role_versions
12739    }
12740
12741    #[tokio::test]
12742    async fn route_open_relays_role_versions_verbatim() {
12743        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12744        assert_eq!(
12745            bind_role_versions_for(Some(sent.clone()), 141).await,
12746            Some(sent)
12747        );
12748    }
12749
12750    /// An empty map declares nothing, so the provider sees no field rather
12751    /// than an empty object it would have to treat as a second "none".
12752    #[tokio::test]
12753    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12754        assert_eq!(bind_role_versions_for(None, 143).await, None);
12755        assert_eq!(
12756            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12757            None
12758        );
12759    }
12760
12761    #[tokio::test]
12762    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12763        let registry = Arc::new(Registry::default());
12764        let forwarding = Arc::new(ForwardingTable::default());
12765        let handler =
12766            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12767        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12768        hello_via_sink(
12769            &handler,
12770            &module_ctx,
12771            &mut module_rx,
12772            hello_frame("aft", PROTOCOL_VERSION, 7),
12773        )
12774        .await;
12775
12776        let nine: BTreeMap<String, String> = (0..9)
12777            .map(|index| (format!("role-{index}"), "v1".to_string()))
12778            .collect();
12779        for (label, malformed) in [
12780            (
12781                "invalid role name",
12782                role_versions(&[("Tool_Provider", "v1")]),
12783            ),
12784            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12785            ("nine entries", nine),
12786        ] {
12787            // A refused open answers at once; one that reached the module
12788            // would wait for its bind ack and trip this timeout.
12789            let responses = tokio::time::timeout(
12790                Duration::from_secs(1),
12791                handler.handle_control_frame(
12792                    &route_ctx(ConnectionId::new(148)).0,
12793                    route_open_frame_with_role_versions(
12794                        404,
12795                        "aft",
12796                        unique_project_root("role-versions-malformed"),
12797                        Some(malformed),
12798                    ),
12799                ),
12800            )
12801            .await
12802            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12803            .unwrap();
12804            assert_eq!(responses.len(), 1, "{label}");
12805            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12806            let error = parse_error(&responses[0]);
12807            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12808            assert_eq!(
12809                error["detail"]["field"], "role_versions",
12810                "{label}: {error}"
12811            );
12812            assert!(
12813                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12814                "{label}: a malformed declaration is terminal"
12815            );
12816            assert!(
12817                module_rx.try_recv().is_err(),
12818                "{label}: the module must never see a bind"
12819            );
12820        }
12821    }
12822
12823    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12824    /// consumer can tell this daemon forwards the field from one that would
12825    /// drop it.
12826    #[tokio::test]
12827    async fn route_role_versions_capability_is_advertised() {
12828        let handler = ControlHandler::new(Arc::new(Registry::default()));
12829        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12830        let ack = hello_via_sink(
12831            &handler,
12832            &ctx,
12833            &mut rx,
12834            hello_frame("m", PROTOCOL_VERSION, 1),
12835        )
12836        .await;
12837        let ack = parse_ack(&ack);
12838        assert!(
12839            ack.subc_capabilities
12840                .iter()
12841                .any(|c| c == "route-role-versions/v1"),
12842            "{:?}",
12843            ack.subc_capabilities
12844        );
12845
12846        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12847        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12848        let reply = handler
12849            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12850            .await
12851            .unwrap()
12852            .pop()
12853            .unwrap();
12854        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12855            serde_json::from_slice(&reply.body).unwrap()
12856        else {
12857            panic!("not a server.describe reply");
12858        };
12859        assert!(
12860            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12861            "{capabilities:?}"
12862        );
12863    }
12864
12865    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12866    /// catalog entry without `events` (the module declares none) from a daemon
12867    /// that never reports declared events.
12868    #[tokio::test]
12869    async fn catalog_events_capability_is_advertised() {
12870        let handler = ControlHandler::new(Arc::new(Registry::default()));
12871        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12872        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12873        let reply = handler
12874            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12875            .await
12876            .unwrap()
12877            .pop()
12878            .unwrap();
12879        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12880            serde_json::from_slice(&reply.body).unwrap()
12881        else {
12882            panic!("not a server.describe reply");
12883        };
12884        assert!(
12885            capabilities
12886                .iter()
12887                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12888            "{capabilities:?}"
12889        );
12890    }
12891
12892    #[tokio::test]
12893    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12894        let handler = ControlHandler::new(Arc::new(Registry::default()));
12895        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12896        let ack = hello_via_sink(
12897            &handler,
12898            &module,
12899            &mut module_rx,
12900            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12901        )
12902        .await;
12903        let ack = parse_ack(&ack);
12904        assert!(
12905            ack.subc_capabilities
12906                .iter()
12907                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12908            "{:?}",
12909            ack.subc_capabilities
12910        );
12911        assert!(ack
12912            .subc_capabilities
12913            .iter()
12914            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12915
12916        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12917        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12918        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12919        let reply = handler
12920            .handle_control_frame(&client, frame)
12921            .await
12922            .unwrap()
12923            .pop()
12924            .unwrap();
12925        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12926            serde_json::from_slice(&reply.body).unwrap()
12927        else {
12928            panic!("not a server.describe reply");
12929        };
12930        assert!(
12931            capabilities
12932                .iter()
12933                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12934            "{capabilities:?}"
12935        );
12936        assert!(capabilities
12937            .iter()
12938            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12939    }
12940
12941    #[tokio::test]
12942    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12943        let handler = ControlHandler::new(Arc::new(Registry::default()));
12944        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12945        let ack = hello_via_sink(
12946            &handler,
12947            &module,
12948            &mut rx,
12949            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12950        )
12951        .await;
12952        assert!(parse_ack(&ack)
12953            .subc_capabilities
12954            .iter()
12955            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12956        assert!(parse_ack(&ack)
12957            .subc_capabilities
12958            .iter()
12959            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12960        let (client, _rx) = route_ctx(ConnectionId::new(157));
12961        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12962        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12963        let reply = handler
12964            .handle_control_frame(&client, frame)
12965            .await
12966            .unwrap()
12967            .pop()
12968            .unwrap();
12969        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12970            serde_json::from_slice(&reply.body).unwrap()
12971        else {
12972            panic!("server.describe reply")
12973        };
12974        assert!(capabilities
12975            .iter()
12976            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12977        assert!(capabilities
12978            .iter()
12979            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12980    }
12981
12982    #[tokio::test]
12983    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12984        let handler = ControlHandler::new(Arc::new(Registry::default()));
12985        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12986        let ack = hello_via_sink(
12987            &handler,
12988            &module,
12989            &mut rx,
12990            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12991        )
12992        .await;
12993        assert!(parse_ack(&ack)
12994            .subc_capabilities
12995            .iter()
12996            .any(|cap| cap == "scope-workdir/v1"));
12997        let (client, _rx) = route_ctx(ConnectionId::new(159));
12998        let frame = Frame::build(
12999            FrameType::Request,
13000            control_flags(),
13001            0,
13002            0,
13003            5,
13004            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13005        )
13006        .unwrap();
13007        let reply = handler
13008            .handle_control_frame(&client, frame)
13009            .await
13010            .unwrap()
13011            .remove(0);
13012        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13013            serde_json::from_slice(&reply.body).unwrap()
13014        else {
13015            panic!("server.describe reply")
13016        };
13017        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13018    }
13019
13020    #[tokio::test]
13021    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13022        let handler = ControlHandler::new(Arc::new(Registry::default()));
13023        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13024        let ack = hello_via_sink(
13025            &handler,
13026            &module,
13027            &mut rx,
13028            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13029        )
13030        .await;
13031        assert!(parse_ack(&ack)
13032            .subc_capabilities
13033            .iter()
13034            .any(|cap| cap == "worker-runs/v1"));
13035        let (client, _rx) = route_ctx(ConnectionId::new(161));
13036        let frame = Frame::build(
13037            FrameType::Request,
13038            control_flags(),
13039            0,
13040            0,
13041            5,
13042            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13043        )
13044        .unwrap();
13045        let reply = handler
13046            .handle_control_frame(&client, frame)
13047            .await
13048            .unwrap()
13049            .remove(0);
13050        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13051            serde_json::from_slice(&reply.body).unwrap()
13052        else {
13053            panic!("server.describe reply")
13054        };
13055        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13056    }
13057
13058    #[tokio::test]
13059    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13060        let handler = ControlHandler::new(Arc::new(Registry::default()));
13061        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13062        let ack = hello_via_sink(
13063            &handler,
13064            &module,
13065            &mut rx,
13066            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13067        )
13068        .await;
13069        assert!(parse_ack(&ack)
13070            .subc_capabilities
13071            .iter()
13072            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13073        let (client, _rx) = route_ctx(ConnectionId::new(155));
13074        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13075        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13076        let reply = handler
13077            .handle_control_frame(&client, frame)
13078            .await
13079            .unwrap()
13080            .pop()
13081            .unwrap();
13082        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13083            serde_json::from_slice(&reply.body).unwrap()
13084        else {
13085            panic!("not a server.describe reply")
13086        };
13087        assert!(capabilities
13088            .iter()
13089            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13090    }
13091
13092    #[tokio::test]
13093    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13094        let registry = Arc::new(Registry::default());
13095        let forwarding = Arc::new(ForwardingTable::default());
13096        let handler =
13097            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13098                .with_health_probe_timeout(Duration::from_secs(5));
13099        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13100        hello_via_sink(
13101            &handler,
13102            &module_ctx,
13103            &mut module_rx,
13104            non_routable_hello_frame_with_control_ops(
13105                "mcp",
13106                300,
13107                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13108            ),
13109        )
13110        .await;
13111        assert!(registry
13112            .get_module("mcp")
13113            .unwrap()
13114            .unwrap()
13115            .manifest
13116            .provides
13117            .is_empty());
13118
13119        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13120        let route_response = handler
13121            .handle_control_frame(
13122                &route_client_ctx,
13123                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13124            )
13125            .await
13126            .unwrap();
13127        assert_eq!(route_response[0].header.ty, FrameType::Error);
13128        assert_eq!(
13129            parse_error(&route_response[0])["code"],
13130            "target_unavailable"
13131        );
13132        assert!(parse_error(&route_response[0])["message"]
13133            .as_str()
13134            .unwrap()
13135            .contains("does not provide the requested target"));
13136        assert!(module_rx.try_recv().is_err());
13137
13138        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13139        let health_handler = handler.clone();
13140        let health_task = tokio::spawn(async move {
13141            health_handler
13142                .handle_control_frame(
13143                    &health_client_ctx,
13144                    supervisor_health_probe_frame(302, "mcp"),
13145                )
13146                .await
13147                .unwrap()
13148        });
13149        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13150            .await
13151            .unwrap()
13152            .unwrap();
13153        assert_eq!(
13154            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13155            ModuleControlRequest::HealthCheck {}
13156        );
13157        handler
13158            .handle_control_frame(
13159                &module_ctx,
13160                health_response(health_frame.header.corr, HealthStatus::Ok),
13161            )
13162            .await
13163            .unwrap();
13164        let health_response = health_task.await.unwrap();
13165        assert_eq!(health_response[0].header.ty, FrameType::Response);
13166        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13167            ClientControlResponse::SupervisorHealthProbe {
13168                module_id, status, ..
13169            } => {
13170                assert_eq!(module_id, "mcp");
13171                assert_eq!(status, HealthStatus::Ok);
13172            }
13173            other => panic!("unexpected health response: {other:?}"),
13174        }
13175
13176        // Exercise the forwarding cleanup path directly while leaving the registry
13177        // advertisement in place. If cleanup leaves a stale control sink behind,
13178        // the next probe will enqueue onto it and wait for the long probe timeout
13179        // instead of returning an immediate no-connection error.
13180        forwarding
13181            .cleanup_connection(module_ctx.connection_id)
13182            .unwrap();
13183        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13184        let cleanup_response = tokio::time::timeout(
13185            Duration::from_millis(200),
13186            handler.handle_control_frame(
13187                &cleanup_probe_ctx,
13188                supervisor_health_probe_frame(303, "mcp"),
13189            ),
13190        )
13191        .await
13192        .expect("probe should fail immediately when the control lane is gone")
13193        .unwrap();
13194        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13195        assert_eq!(
13196            parse_error(&cleanup_response[0])["code"],
13197            "target_unavailable"
13198        );
13199        assert!(parse_error(&cleanup_response[0])["message"]
13200            .as_str()
13201            .unwrap()
13202            .contains("no module connection"));
13203
13204        handler
13205            .cleanup_connection(module_ctx.connection_id)
13206            .unwrap();
13207    }
13208
13209    #[tokio::test]
13210    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13211        let registry = Arc::new(Registry::default());
13212        let supervisor_handle = SupervisorHandle::new();
13213        let supervisor =
13214            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13215                .with_handle(supervisor_handle.clone())
13216                .with_connection_file_path(
13217                    std::env::temp_dir()
13218                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13219                );
13220        let module = supervisor
13221            .supervise_configured(
13222                ModuleSpec {
13223                    module_id: "warming".to_string(),
13224                    program: fake_aft_stub_path(),
13225                    args: Vec::new(),
13226                    env: Vec::new(),
13227                    reserved: false,
13228                    reserved_prefixes: Vec::new(),
13229                    protocol: ModuleProtocol::Subc,
13230                    overlap: Default::default(),
13231                },
13232                true,
13233            )
13234            .unwrap();
13235        assert_eq!(module.state().unwrap(), ModuleState::Running);
13236
13237        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13238        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13239        let response = handler
13240            .handle_control_frame(
13241                &ctx,
13242                route_open_frame(304, "warming", unique_project_root("warming")),
13243            )
13244            .await
13245            .unwrap();
13246        module.stop().await.unwrap();
13247
13248        assert_eq!(response[0].header.ty, FrameType::Error);
13249        let error = parse_error(&response[0]);
13250        assert_eq!(error["code"], "module_warming");
13251        assert!(error["message"]
13252            .as_str()
13253            .unwrap()
13254            .contains("state=running, enabled=true, live=false"));
13255    }
13256
13257    #[test]
13258    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13259        let handler = ControlHandler::new(Arc::new(Registry::default()));
13260        let capture = EventCapture::default();
13261        let _subscriber =
13262            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13263        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13264        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13265        let pending = (0..limit).collect::<Vec<_>>();
13266        let response = handler
13267            .route_open_capacity_refusal(
13268                &ctx,
13269                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13270                "busy",
13271                pending.len(),
13272                limit,
13273            )
13274            .unwrap();
13275        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13276        let event = capture
13277            .events()
13278            .into_iter()
13279            .find(|event| {
13280                event.target == "control"
13281                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13282            })
13283            .expect("connection admission refusal event");
13284        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13285        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13286        // Refused before `handle_route_open` parses the request, so the caller
13287        // fields are read from the frame itself.
13288        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13289        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13290    }
13291
13292    #[test]
13293    fn route_open_target_cap_logs_admission_reason_and_capacity() {
13294        let handler = ControlHandler::new(Arc::new(Registry::default()));
13295        let capture = EventCapture::default();
13296        let _subscriber =
13297            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13298        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13299        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13300        let guards = (0..limit)
13301            .map(|_| {
13302                handler
13303                    .route_bind_concurrency
13304                    .try_admit("busy", limit)
13305                    .unwrap()
13306            })
13307            .collect::<Vec<_>>();
13308        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13309            Err(in_flight) => in_flight,
13310            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13311        };
13312        let response = handler
13313            .route_open_target_capacity_refusal(
13314                &ctx,
13315                &RouteOpenCaller::unknown(),
13316                &route_open_frame(397, "busy", unique_project_root("target-cap")),
13317                "busy",
13318                in_flight,
13319            )
13320            .unwrap();
13321        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13322        let event = capture
13323            .events()
13324            .into_iter()
13325            .find(|event| {
13326                event.target == "control"
13327                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13328            })
13329            .expect("target admission refusal event");
13330        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13331        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13332        drop(guards);
13333    }
13334
13335    #[cfg(unix)]
13336    #[tokio::test(start_paused = true)]
13337    async fn supervisor_list_shows_a_failed_modules_next_retry() {
13338        let registry = Arc::new(Registry::default());
13339        let handle = SupervisorHandle::new();
13340        let supervisor =
13341            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13342                .with_handle(handle.clone());
13343        let module = supervisor
13344            .spawn(crate::ModuleSpec {
13345                module_id: "retry-list".into(),
13346                program: PathBuf::from("/bin/sh"),
13347                args: vec!["-c".into(), "exit 17".into()],
13348                env: vec![],
13349                reserved: false,
13350                reserved_prefixes: vec![],
13351                protocol: ModuleProtocol::None,
13352                overlap: Default::default(),
13353            })
13354            .unwrap();
13355        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13356        while module.state().unwrap() != crate::ModuleState::Failed {
13357            assert!(
13358                std::time::Instant::now() < deadline,
13359                "waiting for budget failure"
13360            );
13361            tokio::task::yield_now().await;
13362        }
13363        let handler = ControlHandler::new(registry).with_supervisor(handle);
13364        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13365        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13366        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13367        assert_eq!(body["modules"][0]["state"], "failed");
13368        let retry = body["modules"][0]["next_retry_at_ms"]
13369            .as_u64()
13370            .expect("scheduled retry on the wire");
13371        let now = std::time::SystemTime::now()
13372            .duration_since(std::time::UNIX_EPOCH)
13373            .unwrap()
13374            .as_millis() as u64;
13375        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13376        module.set_enabled(false).await.unwrap();
13377        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13378        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13379        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13380        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13381    }
13382
13383    #[tokio::test]
13384    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13385        let registry = Arc::new(Registry::default());
13386        let handle = SupervisorHandle::new();
13387        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13388            .with_handle(handle.clone());
13389        supervisor
13390            .supervise_configured(
13391                crate::ModuleSpec {
13392                    module_id: "drift".into(),
13393                    program: PathBuf::from("disabled-module"),
13394                    args: Vec::new(),
13395                    env: Vec::new(),
13396                    reserved: false,
13397                    reserved_prefixes: Vec::new(),
13398                    protocol: ModuleProtocol::Subc,
13399                    overlap: Default::default(),
13400                },
13401                false,
13402            )
13403            .unwrap();
13404        let handler = ControlHandler::new(registry).with_supervisor(handle);
13405        for configured in [None, Some(false), Some(true)] {
13406            handler.capability_evaluator.configure(
13407                configured.map(|enabled| ("drift".to_string(), enabled)),
13408                BTreeMap::new(),
13409            );
13410            let frame =
13411                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13412            let replies = handler
13413                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13414                .await
13415                .unwrap();
13416            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13417            assert!(body.get("configured_enabled").is_none());
13418            assert_eq!(body["op"], "supervisor.ack");
13419            let frame =
13420                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13421            let replies = handler.handle_supervisor_list(frame).await.unwrap();
13422            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13423            assert_eq!(
13424                body["modules"][0].get("configured_enabled"),
13425                configured.map(Value::Bool).as_ref()
13426            );
13427        }
13428    }
13429
13430    #[tokio::test(flavor = "current_thread")]
13431    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13432        let handler = ControlHandler::new(Arc::new(Registry::default()));
13433        let capture = EventCapture::default();
13434        let _subscriber =
13435            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13436        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13437        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13438
13439        let _ = handler
13440            .handle_client_control_request(
13441                &ctx,
13442                frame,
13443                ClientControlRequest::SupervisorSetEnabled {
13444                    module_id: "broca".to_string(),
13445                    enabled: false,
13446                },
13447                &mut None,
13448            )
13449            .await
13450            .unwrap();
13451
13452        let events = capture
13453            .events()
13454            .into_iter()
13455            .filter(|event| {
13456                event.target == "control"
13457                    && event.fields.get("message").map(String::as_str)
13458                        == Some("supervisor request received")
13459            })
13460            .collect::<Vec<_>>();
13461        assert_eq!(events.len(), 1, "one supervisor request log line");
13462        let event = &events[0];
13463        assert_eq!(event.level, tracing::Level::INFO);
13464        assert_eq!(
13465            event.fields.get("op"),
13466            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13467        );
13468        assert_eq!(
13469            event.fields.get("module_id"),
13470            Some(&"\"broca\"".to_string())
13471        );
13472        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13473        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13474        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13475    }
13476
13477    #[tokio::test(flavor = "current_thread")]
13478    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13479        let registry = Arc::new(Registry::default());
13480        let module_connection = ConnectionId::new(303);
13481        registry
13482            .register_with_control_ops(
13483                manifest("aft", PROTOCOL_VERSION),
13484                PROTOCOL_VERSION,
13485                module_connection,
13486                module_baseline_control_ops(),
13487            )
13488            .unwrap();
13489        let handler = ControlHandler::new(registry);
13490        let capture = EventCapture::default();
13491        let _subscriber =
13492            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13493        let (ctx, _rx) = route_ctx(module_connection);
13494        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13495
13496        let _ = handler
13497            .handle_client_control_request(
13498                &ctx,
13499                frame,
13500                ClientControlRequest::SupervisorSetEnabled {
13501                    module_id: "broca".to_string(),
13502                    enabled: false,
13503                },
13504                &mut None,
13505            )
13506            .await
13507            .unwrap();
13508
13509        let callers = capture
13510            .events()
13511            .into_iter()
13512            .filter(|event| {
13513                event.fields.get("message").map(String::as_str)
13514                    == Some("supervisor request received")
13515            })
13516            .map(|event| event.fields.get("caller").cloned())
13517            .collect::<Vec<_>>();
13518        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13519    }
13520
13521    #[tokio::test(flavor = "current_thread")]
13522    async fn supervisor_list_does_not_log_request_received() {
13523        let handler = ControlHandler::new(Arc::new(Registry::default()));
13524        let capture = EventCapture::default();
13525        let _subscriber =
13526            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13527        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13528        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13529
13530        handler
13531            .handle_client_control_request(
13532                &ctx,
13533                frame,
13534                ClientControlRequest::SupervisorList {},
13535                &mut None,
13536            )
13537            .await
13538            .unwrap();
13539
13540        assert!(capture.events().into_iter().all(|event| {
13541            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13542        }));
13543    }
13544
13545    #[tokio::test(flavor = "current_thread")]
13546    async fn supervisor_rescan_preview_does_not_log_request_received() {
13547        let handler = ControlHandler::new(Arc::new(Registry::default()));
13548        let capture = EventCapture::default();
13549        let _subscriber =
13550            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13551        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13552        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13553
13554        handler
13555            .handle_client_control_request(
13556                &ctx,
13557                frame,
13558                ClientControlRequest::SupervisorRescan { preview: true },
13559                &mut None,
13560            )
13561            .await
13562            .unwrap();
13563
13564        assert!(capture.events().into_iter().all(|event| {
13565            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13566        }));
13567    }
13568
13569    struct ApproveOperator;
13570    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13571        fn prompt(
13572            &self,
13573            _: &str,
13574            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13575        ) -> crate::operator_confirm::ProviderResult {
13576            crate::operator_confirm::ProviderResult::Approved
13577        }
13578    }
13579
13580    #[tokio::test]
13581    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13582        assert_eq!(
13583            ControlHandler::default().privileged_operator_prompt,
13584            PrivilegedOperatorPrompt::Log
13585        );
13586        for (initial, changed, running) in [
13587            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13588            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13589        ] {
13590            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13591            let path = dir.join("subc.jsonc");
13592            std::fs::write(
13593                &path,
13594                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13595            )
13596            .unwrap();
13597            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13598            let registry = Arc::new(Registry::default());
13599            let handle = SupervisorHandle::new();
13600            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13601                .with_handle(handle.clone());
13602            let handler = ControlHandler::new(registry)
13603                .with_supervisor(handle)
13604                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13605                .with_supervisor_rescan(supervisor, &path, config.port);
13606            handler.forwarding.operator_confirms().configure(
13607                Arc::new(ApproveOperator),
13608                crate::operator_confirm::OperatorLimits::default(),
13609            );
13610            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13611
13612            for (disk_mode, needs_restart) in [
13613                (Some(initial), false),
13614                (Some(changed), true),
13615                (Some(changed), true),
13616                (None, running == PrivilegedOperatorPrompt::Enforce),
13617                (Some(initial), false),
13618            ] {
13619                let mut doc = json!({"version":1});
13620                if let Some(mode) = disk_mode {
13621                    doc["privileged_operator_prompt"] = json!(mode);
13622                }
13623                std::fs::write(&path, doc.to_string()).unwrap();
13624                for preview in [true, false] {
13625                    let request = ClientControlRequest::SupervisorRescan { preview };
13626                    let frame = Frame::build(
13627                        FrameType::Request,
13628                        control_flags(),
13629                        0,
13630                        0,
13631                        1,
13632                        serde_json::to_vec(&request).unwrap(),
13633                    )
13634                    .unwrap();
13635                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13636                    assert_eq!(frames.len(), 1);
13637                    let ClientControlResponse::SupervisorRescan { result } =
13638                        serde_json::from_slice(&frames[0].body).unwrap()
13639                    else {
13640                        panic!("expected a rescan response");
13641                    };
13642                    let expected = if needs_restart {
13643                        vec!["privileged_operator_prompt"]
13644                    } else {
13645                        Vec::new()
13646                    };
13647                    assert_eq!(result.restart_required, expected);
13648                    assert_eq!(handler.privileged_operator_prompt, running);
13649                    assert_eq!(
13650                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13651                        running
13652                    );
13653                }
13654            }
13655        }
13656    }
13657
13658    #[tokio::test]
13659    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13660        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13661        let path = dir.join("subc.jsonc");
13662        let registry = Arc::new(Registry::default());
13663        let handle = SupervisorHandle::new();
13664        let supervisor =
13665            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13666        let handler = ControlHandler::new(registry)
13667            .with_supervisor(handle)
13668            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13669            .with_supervisor_rescan(supervisor, &path, None);
13670        handler.forwarding.operator_confirms().configure(
13671            Arc::new(ApproveOperator),
13672            crate::operator_confirm::OperatorLimits::default(),
13673        );
13674        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13675        for preview in [true, false] {
13676            std::fs::write(
13677                &path,
13678                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13679            )
13680            .unwrap();
13681            let request = ClientControlRequest::SupervisorRescan { preview };
13682            let frame = Frame::build(
13683                FrameType::Request,
13684                control_flags(),
13685                0,
13686                0,
13687                1,
13688                serde_json::to_vec(&request).unwrap(),
13689            )
13690            .unwrap();
13691            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13692            assert_eq!(frames.len(), 1);
13693            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13694            assert_eq!(
13695                handler.privileged_operator_prompt,
13696                PrivilegedOperatorPrompt::Enforce
13697            );
13698        }
13699    }
13700
13701    /// One wire code has several senders, so the refusal line names the check
13702    /// that refused. This drives the shared refusal path for ordinary refusals
13703    /// with an unregistered
13704    /// target and requires the branch label on the event.
13705    #[tokio::test]
13706    async fn route_open_refusal_names_the_check_that_refused() {
13707        let handler = ControlHandler::new(Arc::new(Registry::default()));
13708        let capture = EventCapture::default();
13709        let _subscriber =
13710            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13711        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13712        let response = handler
13713            .handle_control_frame(
13714                &ctx,
13715                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13716            )
13717            .await
13718            .unwrap();
13719
13720        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13721        let event = capture
13722            .events()
13723            .into_iter()
13724            .find(|event| {
13725                event.target == "control"
13726                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13727            })
13728            .expect("route.open refusal event");
13729        assert_eq!(
13730            event.fields.get("reason"),
13731            Some(&"\"not_registered\"".to_string())
13732        );
13733    }
13734
13735    #[tokio::test]
13736    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13737        let registry = Arc::new(Registry::default());
13738        let supervisor_handle = SupervisorHandle::new();
13739        let supervisor =
13740            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13741                .with_handle(supervisor_handle.clone())
13742                .with_connection_file_path(std::env::temp_dir().join(format!(
13743                    "subc-route-open-refusal-info-{}",
13744                    std::process::id()
13745                )));
13746        let module = supervisor
13747            .supervise_configured(
13748                ModuleSpec {
13749                    module_id: "warming".to_string(),
13750                    program: fake_aft_stub_path(),
13751                    args: Vec::new(),
13752                    env: Vec::new(),
13753                    reserved: false,
13754                    reserved_prefixes: Vec::new(),
13755                    protocol: ModuleProtocol::Subc,
13756                    overlap: Default::default(),
13757                },
13758                true,
13759            )
13760            .unwrap();
13761        assert_eq!(module.state().unwrap(), ModuleState::Running);
13762
13763        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13764        assert!(handler
13765            .counters()
13766            .snapshot()
13767            .get("route_open_refused_by_code")
13768            .is_none());
13769        let capture = EventCapture::default();
13770        let _subscriber =
13771            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13772        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13773        let response = handler
13774            .handle_control_frame(
13775                &ctx,
13776                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13777            )
13778            .await
13779            .unwrap();
13780        module.stop().await.unwrap();
13781
13782        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13783        let event = capture
13784            .events()
13785            .into_iter()
13786            .find(|event| {
13787                event.target == "control"
13788                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13789            })
13790            .expect("route.open refusal event");
13791        assert_eq!(
13792            event.fields.get("module_id"),
13793            Some(&"\"warming\"".to_string())
13794        );
13795        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13796        assert_eq!(
13797            event.fields.get("reason"),
13798            Some(&"\"supervised_not_registered\"".to_string())
13799        );
13800        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13801        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13802        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13803        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13804        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13805        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13806        assert_eq!(
13807            handler.counters().snapshot()["route_open_refused_by_code"],
13808            json!({ "module_warming": 1 })
13809        );
13810    }
13811
13812    const OUTAGE_START: &str = "route.open refusing module: not serving";
13813    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13814
13815    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13816        capture
13817            .events()
13818            .into_iter()
13819            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13820            .collect()
13821    }
13822
13823    fn supervise_stub(
13824        registry: &Arc<Registry>,
13825        module_id: &str,
13826        enabled: bool,
13827    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13828        let supervisor_handle = SupervisorHandle::new();
13829        let supervisor =
13830            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13831                .with_handle(supervisor_handle.clone())
13832                .with_connection_file_path(std::env::temp_dir().join(format!(
13833                    "subc-route-outage-{module_id}-{}",
13834                    std::process::id()
13835                )));
13836        let module = supervisor
13837            .supervise_configured(
13838                ModuleSpec {
13839                    module_id: module_id.to_string(),
13840                    program: fake_aft_stub_path(),
13841                    args: Vec::new(),
13842                    env: Vec::new(),
13843                    reserved: false,
13844                    reserved_prefixes: Vec::new(),
13845                    protocol: ModuleProtocol::Subc,
13846                    overlap: Default::default(),
13847                },
13848                enabled,
13849            )
13850            .unwrap();
13851        (supervisor_handle, module)
13852    }
13853
13854    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13855        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13856            module_id: module_id.to_string(),
13857            drain_timeout_ms: Some(50),
13858        })
13859        .unwrap();
13860        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13861    }
13862
13863    /// Two handlers built over one forwarding table must share one outage
13864    /// tracker; separate trackers would each log their own opening line for
13865    /// the same outage.
13866    #[test]
13867    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13868        let registry = Arc::new(Registry::default());
13869        let forwarding = Arc::new(ForwardingTable::default());
13870        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13871        let second = ControlHandler::with_forwarding(registry, forwarding);
13872        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13873    }
13874
13875    /// A client can name any module id it likes. Refusing an unknown one,
13876    /// however often, must not create outage state or outage lines, or the
13877    /// tracker would be a memory sink any client could fill.
13878    #[tokio::test(flavor = "current_thread")]
13879    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13880        let handler = ControlHandler::new(Arc::new(Registry::default()));
13881        let capture = EventCapture::default();
13882        let _subscriber =
13883            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13884        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13885        for corr in 0..8 {
13886            let response = handler
13887                .handle_control_frame(
13888                    &ctx,
13889                    route_open_frame(
13890                        380 + corr,
13891                        &format!("nobody-{corr}"),
13892                        unique_project_root("outage-unknown"),
13893                    ),
13894                )
13895                .await
13896                .unwrap();
13897            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13898        }
13899
13900        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13901        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13902        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13903    }
13904
13905    /// Drives the refusal path end to end: a supervised module that served
13906    /// before and stopped being registered with no instruction to stop is a
13907    /// WARN, and the same module refused after an operator `supervisor.restart`
13908    /// is an INFO.
13909    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13910    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13911        let registry = Arc::new(Registry::default());
13912        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13913        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13914        let capture = EventCapture::default();
13915        let _subscriber =
13916            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13917        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13918        // The stub never registers, so pretend it served once: otherwise every
13919        // refusal would fall in its startup window.
13920        handler.route_outages.record_accepted("outage-restart");
13921
13922        let response = handler
13923            .handle_control_frame(
13924                &ctx,
13925                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13926            )
13927            .await
13928            .unwrap();
13929        assert_eq!(response[0].header.ty, FrameType::Error);
13930        let starts = outage_lines(&capture, OUTAGE_START);
13931        assert_eq!(starts.len(), 1, "{starts:?}");
13932        assert_eq!(starts[0].level, tracing::Level::WARN);
13933        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13934        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13935        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13936        handler.route_outages.record_accepted("outage-restart");
13937        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13938
13939        let restart = handler
13940            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13941            .await
13942            .unwrap();
13943        assert_eq!(
13944            restart[0].header.ty,
13945            FrameType::Response,
13946            "{:?}",
13947            parse_error(&restart[0])
13948        );
13949        handler
13950            .handle_control_frame(
13951                &ctx,
13952                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13953            )
13954            .await
13955            .unwrap();
13956        module.stop().await.unwrap();
13957
13958        let starts = outage_lines(&capture, OUTAGE_START);
13959        assert_eq!(starts.len(), 2, "{starts:?}");
13960        assert_eq!(starts[1].level, tracing::Level::INFO);
13961        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13962    }
13963
13964    /// A restart refused before it touched the module (here: the module is
13965    /// disabled) must clear its operator mark, so the next real outage is
13966    /// still reported as a warning.
13967    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13968    async fn failed_operator_restart_leaves_no_operator_mark() {
13969        let registry = Arc::new(Registry::default());
13970        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13971        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13972        let capture = EventCapture::default();
13973        let _subscriber =
13974            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13975        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13976        handler.route_outages.record_accepted("outage-disabled");
13977
13978        let restart = handler
13979            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13980            .await
13981            .unwrap();
13982        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13983        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13984
13985        handler
13986            .handle_control_frame(
13987                &ctx,
13988                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13989            )
13990            .await
13991            .unwrap();
13992        let starts = outage_lines(&capture, OUTAGE_START);
13993        assert_eq!(starts.len(), 1, "{starts:?}");
13994        assert_eq!(starts[0].level, tracing::Level::WARN);
13995    }
13996
13997    #[tokio::test(flavor = "current_thread")]
13998    async fn route_open_unknown_module_escapes_target_module_id() {
13999        let handler = ControlHandler::new(Arc::new(Registry::default()));
14000        let capture = EventCapture::default();
14001        let _subscriber =
14002            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14003        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14004        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14005        let response = handler
14006            .handle_control_frame(
14007                &ctx,
14008                route_open_frame(
14009                    395,
14010                    hostile_module_id,
14011                    unique_project_root("hostile-target-module-id"),
14012                ),
14013            )
14014            .await
14015            .unwrap();
14016
14017        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14018        let event = capture
14019            .events()
14020            .into_iter()
14021            .find(|event| {
14022                event.target == "control"
14023                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14024            })
14025            .expect("route.open unknown-module refusal event");
14026        let logged = event.fields.get("module_id").expect("module_id field");
14027        assert!(!logged.bytes().any(|byte| byte < 0x20));
14028        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14029    }
14030
14031    /// A route.open refusal is the daemon's own answer, so its ERROR carries
14032    /// the daemon-origin bit on the frame the client receives.
14033    #[tokio::test(flavor = "current_thread")]
14034    async fn route_open_refusal_carries_daemon_origin() {
14035        let handler = ControlHandler::new(Arc::new(Registry::default()));
14036        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14037        let response = handler
14038            .handle_control_frame(
14039                &ctx,
14040                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14041            )
14042            .await
14043            .unwrap();
14044
14045        assert_eq!(response.len(), 1);
14046        assert_eq!(response[0].header.ty, FrameType::Error);
14047        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14048        assert!(
14049            response[0].header.flags.is_daemon_origin(),
14050            "route.open refusal must carry daemon origin: {:?}",
14051            response[0].header.flags
14052        );
14053    }
14054
14055    /// The drain `module_reloading` refusal is daemon-synthesized for an
14056    /// endpoint that is reloading, so it carries the daemon-origin bit on the
14057    /// frame the client receives.
14058    #[tokio::test(flavor = "current_thread")]
14059    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14060        let registry = Arc::new(Registry::default());
14061        let forwarding = Arc::new(ForwardingTable::default());
14062        let handler =
14063            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14064        let module_connection = ConnectionId::new(95);
14065        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14066        hello_via_sink(
14067            &handler,
14068            &module_ctx,
14069            &mut module_rx,
14070            hello_frame("aft", PROTOCOL_VERSION, 395),
14071        )
14072        .await;
14073        forwarding
14074            .begin_module_drain("aft", RouteCloseReason::Restart)
14075            .unwrap()
14076            .expect("the module endpoint drains");
14077
14078        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14079        let response = handler
14080            .handle_control_frame(
14081                &client_ctx,
14082                route_open_frame(396, "aft", unique_project_root("drain-origin")),
14083            )
14084            .await
14085            .unwrap();
14086
14087        assert_eq!(response.len(), 1);
14088        assert_eq!(response[0].header.ty, FrameType::Error);
14089        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14090        assert!(
14091            response[0].header.flags.is_daemon_origin(),
14092            "drain module_reloading must carry daemon origin: {:?}",
14093            response[0].header.flags
14094        );
14095    }
14096
14097    /// A module that refuses a bind with a daemon code still cannot mint a
14098    /// daemon refusal: the relayed `module_reloading` body reaches the client
14099    /// with the daemon-origin bit clear.
14100    #[tokio::test(flavor = "current_thread")]
14101    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14102        let registry = Arc::new(Registry::default());
14103        let forwarding = Arc::new(ForwardingTable::default());
14104        let handler =
14105            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14106        let module_connection = ConnectionId::new(95);
14107        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14108        hello_via_sink(
14109            &handler,
14110            &module_ctx,
14111            &mut module_rx,
14112            hello_frame("aft", PROTOCOL_VERSION, 395),
14113        )
14114        .await;
14115
14116        let client_connection = ConnectionId::new(96);
14117        let (client_ctx, _client_rx) = route_ctx(client_connection);
14118        let (route_task, bind) = relay_route_open(
14119            &handler,
14120            client_connection,
14121            &client_ctx.egress,
14122            &mut module_rx,
14123            396,
14124            "aft",
14125            "spoofed-reloading",
14126        )
14127        .await;
14128        // The module sets the origin bit on its own rejection too: neither
14129        // the bit nor the daemon code may survive the relay as daemon-owned.
14130        let rejection = Frame::build(
14131            FrameType::Error,
14132            crate::router::daemon_error_flags(),
14133            0,
14134            0,
14135            bind.header.corr,
14136            serde_json::to_vec(&ErrorBody::new(
14137                "module_reloading",
14138                "module refused route.bind",
14139            ))
14140            .unwrap(),
14141        )
14142        .unwrap();
14143        handler
14144            .handle_control_frame(&module_ctx, rejection)
14145            .await
14146            .unwrap();
14147
14148        let response = route_task.await.unwrap();
14149        assert_eq!(response.len(), 1);
14150        assert_eq!(response[0].header.ty, FrameType::Error);
14151        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14152        assert!(
14153            !response[0].header.flags.is_daemon_origin(),
14154            "a module's own rejection must not carry daemon origin: {:?}",
14155            response[0].header.flags
14156        );
14157    }
14158
14159    #[tokio::test(flavor = "current_thread")]
14160    async fn route_open_module_rejection_uses_daemon_counter_key() {
14161        let registry = Arc::new(Registry::default());
14162        let forwarding = Arc::new(ForwardingTable::default());
14163        let handler =
14164            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14165        let module_connection = ConnectionId::new(95);
14166        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14167        hello_via_sink(
14168            &handler,
14169            &module_ctx,
14170            &mut module_rx,
14171            hello_frame("aft", PROTOCOL_VERSION, 395),
14172        )
14173        .await;
14174
14175        let client_connection = ConnectionId::new(96);
14176        let (client_ctx, _client_rx) = route_ctx(client_connection);
14177        let capture = EventCapture::default();
14178        let _subscriber =
14179            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14180        let (route_task, bind) = relay_route_open(
14181            &handler,
14182            client_connection,
14183            &client_ctx.egress,
14184            &mut module_rx,
14185            396,
14186            "aft",
14187            "hostile-module-code",
14188        )
14189        .await;
14190        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14191        let rejection = Frame::build(
14192            FrameType::Error,
14193            control_flags(),
14194            0,
14195            0,
14196            bind.header.corr,
14197            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14198        )
14199        .unwrap();
14200        handler
14201            .handle_control_frame(&module_ctx, rejection)
14202            .await
14203            .unwrap();
14204
14205        let response = route_task.await.unwrap();
14206        assert_eq!(parse_error(&response[0])["code"], hostile_code);
14207        let counters = handler.counters().snapshot();
14208        assert_eq!(
14209            counters["route_open_refused_by_code"],
14210            json!({ "module_rejected": 1 })
14211        );
14212        assert!(counters["route_open_refused_by_code"]
14213            .get(hostile_code)
14214            .is_none());
14215
14216        let event = capture
14217            .events()
14218            .into_iter()
14219            .find(|event| {
14220                event.target == "control"
14221                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14222            })
14223            .expect("route.open module-rejection refusal event");
14224        let logged = event.fields.get("module_code").expect("module_code field");
14225        assert!(!logged.bytes().any(|byte| byte < 0x20));
14226        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14227    }
14228
14229    #[tokio::test]
14230    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14231        let registry = Arc::new(Registry::default());
14232        let supervisor_handle = SupervisorHandle::new();
14233        let missing_program = std::env::temp_dir().join(format!(
14234            "subc-route-open-missing-program-{}",
14235            std::process::id()
14236        ));
14237        let supervisor =
14238            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14239                .with_handle(supervisor_handle.clone());
14240        let module = supervisor
14241            .supervise_configured(
14242                ModuleSpec {
14243                    module_id: "failed".to_string(),
14244                    program: missing_program,
14245                    args: Vec::new(),
14246                    env: Vec::new(),
14247                    reserved: false,
14248                    reserved_prefixes: Vec::new(),
14249                    protocol: ModuleProtocol::Subc,
14250                    overlap: Default::default(),
14251                },
14252                true,
14253            )
14254            .unwrap();
14255        assert_eq!(module.state().unwrap(), ModuleState::Failed);
14256
14257        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14258        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14259        let response = handler
14260            .handle_control_frame(
14261                &ctx,
14262                route_open_frame(305, "failed", unique_project_root("failed")),
14263            )
14264            .await
14265            .unwrap();
14266
14267        assert_eq!(response[0].header.ty, FrameType::Error);
14268        let error = parse_error(&response[0]);
14269        assert_eq!(error["code"], "target_unavailable");
14270        assert!(error["message"]
14271            .as_str()
14272            .unwrap()
14273            .contains("state=failed, enabled=true, live=false"));
14274    }
14275
14276    #[tokio::test]
14277    async fn route_open_role_mismatch_remains_target_unavailable() {
14278        let registry = Arc::new(Registry::default());
14279        let handler = ControlHandler::new(Arc::clone(&registry));
14280        handler
14281            .handle_control(
14282                ConnectionId::new(41),
14283                non_routable_hello_frame_with_control_ops("health-only", 306, None),
14284            )
14285            .unwrap();
14286
14287        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14288        let response = handler
14289            .handle_control_frame(
14290                &ctx,
14291                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14292            )
14293            .await
14294            .unwrap();
14295
14296        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14297        assert!(parse_error(&response[0])["message"]
14298            .as_str()
14299            .unwrap()
14300            .contains("does not provide the requested target"));
14301    }
14302
14303    #[tokio::test]
14304    async fn route_open_inactive_registration_remains_target_unavailable() {
14305        let registry = Arc::new(Registry::default());
14306        let handler = ControlHandler::new(Arc::clone(&registry));
14307        handler
14308            .handle_control(
14309                ConnectionId::new(43),
14310                hello_frame("inactive", PROTOCOL_VERSION, 308),
14311            )
14312            .unwrap();
14313        assert!(registry
14314            .set_module_state_for_test("inactive", ChannelState::Closed)
14315            .unwrap());
14316
14317        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14318        let response = handler
14319            .handle_control_frame(
14320                &ctx,
14321                route_open_frame(309, "inactive", unique_project_root("inactive")),
14322            )
14323            .await
14324            .unwrap();
14325
14326        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14327        assert!(parse_error(&response[0])["message"]
14328            .as_str()
14329            .unwrap()
14330            .contains("is not active"));
14331    }
14332
14333    #[tokio::test]
14334    async fn late_health_reply_is_recorded_through_the_module_response_path() {
14335        let registry = Arc::new(Registry::default());
14336        let forwarding = Arc::new(ForwardingTable::default());
14337        let supervisor_handle = SupervisorHandle::new();
14338        let supervisor =
14339            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
14340                .with_forwarding(Arc::clone(&forwarding))
14341                .with_handle(supervisor_handle.clone());
14342        let module = supervisor
14343            .supervise_configured(
14344                crate::ModuleSpec {
14345                    module_id: "late-health-response".to_string(),
14346                    program: PathBuf::from("disabled-module"),
14347                    args: Vec::new(),
14348                    env: Vec::new(),
14349                    reserved: false,
14350                    reserved_prefixes: Vec::new(),
14351                    protocol: ModuleProtocol::Subc,
14352                    overlap: Default::default(),
14353                },
14354                false,
14355            )
14356            .unwrap();
14357        let handler =
14358            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14359                .with_supervisor(supervisor_handle);
14360        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14361        handler
14362            .handle_control_frame(
14363                &module_ctx,
14364                hello_frame_with_control_ops(
14365                    "late-health-response",
14366                    PROTOCOL_VERSION,
14367                    7,
14368                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14369                ),
14370            )
14371            .await
14372            .unwrap();
14373        let probe_started_at = Instant::now() - Duration::from_millis(80);
14374        let pending = forwarding
14375            .begin_health_probe_rpc_for(
14376                "late-health-response",
14377                MODULE_CONTROL_OP_HEALTH_CHECK,
14378                probe_started_at,
14379                Instant::now() - Duration::from_millis(1),
14380            )
14381            .unwrap();
14382        assert!(forwarding
14383            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14384            .unwrap());
14385
14386        let responses = handler
14387            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14388            .await
14389            .unwrap();
14390
14391        assert!(responses.is_empty());
14392        let health = module.status().unwrap().health;
14393        assert_eq!(health.late_answer_count, 1);
14394        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14395    }
14396
14397    #[tokio::test]
14398    async fn health_probe_timeout_and_module_death_are_typed() {
14399        let registry = Arc::new(Registry::default());
14400        let forwarding = Arc::new(ForwardingTable::default());
14401        let handler =
14402            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14403                .with_health_probe_timeout(Duration::from_millis(50));
14404        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14405        hello_via_sink(
14406            &handler,
14407            &module_ctx,
14408            &mut module_rx,
14409            hello_frame_with_control_ops(
14410                "aft",
14411                PROTOCOL_VERSION,
14412                7,
14413                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14414            ),
14415        )
14416        .await;
14417
14418        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14419        let responses = handler
14420            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14421            .await
14422            .unwrap();
14423        assert_eq!(responses[0].header.ty, FrameType::Error);
14424        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14425        let _ = module_rx.try_recv();
14426
14427        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14428        let health_handler = handler.clone();
14429        let death_task = tokio::spawn(async move {
14430            health_handler
14431                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14432                .await
14433                .unwrap()
14434        });
14435        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14436            .await
14437            .unwrap()
14438            .unwrap();
14439        handler
14440            .cleanup_connection(module_ctx.connection_id)
14441            .unwrap();
14442        let responses = death_task.await.unwrap();
14443        assert_eq!(responses[0].header.ty, FrameType::Error);
14444        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14445    }
14446
14447    #[test]
14448    fn hello_requires_exact_protocol_version() {
14449        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14450            let registry = Arc::new(Registry::default());
14451            let handler = ControlHandler::new(Arc::clone(&registry));
14452            let responses = handler
14453                .handle_control(
14454                    ConnectionId::new(connection),
14455                    hello_frame("aft", offered, 9),
14456                )
14457                .unwrap();
14458
14459            assert_eq!(responses.len(), 1);
14460            assert_eq!(responses[0].header.ty, FrameType::Error);
14461            let error = parse_error(&responses[0]);
14462            assert_eq!(error["code"], "version_unsupported");
14463            assert!(registry.get_module("aft").unwrap().is_none());
14464            assert_eq!(registry.active_registration_count().unwrap(), 0);
14465        }
14466    }
14467
14468    #[test]
14469    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14470        let registry = Arc::new(Registry::default());
14471        let forwarding = Arc::new(ForwardingTable::default());
14472        let handler =
14473            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14474        let module_connection = ConnectionId::new(301);
14475        let registration = registry
14476            .register_with_control_ops(
14477                manifest("aft-push", PROTOCOL_VERSION),
14478                PROTOCOL_VERSION,
14479                module_connection,
14480                module_baseline_control_ops(),
14481            )
14482            .unwrap();
14483        let (module_tx, _module_rx) = mpsc::channel(8);
14484        let endpoint = forwarding
14485            .register_module_connection(
14486                module_connection,
14487                "aft-push".to_string(),
14488                PROTOCOL_VERSION,
14489                manifest_concurrency(&registration.manifest),
14490                FrameSink::new(module_tx),
14491            )
14492            .unwrap();
14493
14494        // A push op this version does not know is ignored (forward-compat), not errored.
14495        let unknown = Frame::build(
14496            FrameType::Push,
14497            control_flags(),
14498            0,
14499            0,
14500            5,
14501            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14502        )
14503        .unwrap();
14504        let out = handler.handle_status_update(endpoint, unknown).unwrap();
14505        assert!(
14506            out.is_empty(),
14507            "unknown push op must be ignored, got {out:?}"
14508        );
14509
14510        // A malformed body for a KNOWN op is a real error worth surfacing.
14511        let malformed = Frame::build(
14512            FrameType::Push,
14513            control_flags(),
14514            0,
14515            0,
14516            6,
14517            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14518        )
14519        .unwrap();
14520        let out = handler.handle_status_update(endpoint, malformed).unwrap();
14521        assert_eq!(out.len(), 1);
14522        assert_eq!(out[0].header.ty, FrameType::Error);
14523        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14524    }
14525
14526    #[test]
14527    fn hello_rejected_when_connection_already_owns_client_routes() {
14528        let registry = Arc::new(Registry::default());
14529        let forwarding = Arc::new(ForwardingTable::default());
14530        let handler =
14531            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14532        // Commits a client route on connection 202 (bound to a module on conn 101).
14533        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
14534        let client_connection = ConnectionId::new(202);
14535
14536        // That same connection now tries to register as a module: rejected, so one
14537        // connection never holds both client-route and module-endpoint state.
14538        let responses = handler
14539            .handle_control(
14540                client_connection,
14541                hello_frame("aft-second", PROTOCOL_VERSION, 9),
14542            )
14543            .unwrap();
14544        assert_eq!(responses[0].header.ty, FrameType::Error);
14545        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14546        assert!(registry.get_module("aft-second").unwrap().is_none());
14547    }
14548
14549    #[tokio::test]
14550    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14551        let registry = Arc::new(Registry::default());
14552        let forwarding = Arc::new(ForwardingTable::default());
14553        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14554        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14555        hello_via_sink(
14556            &handler,
14557            &module_ctx,
14558            &mut module_rx,
14559            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14560        )
14561        .await;
14562        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14563        let pending = forwarding
14564            .begin_route_bind_relay_for_test(
14565                client_ctx.connection_id,
14566                client_ctx.egress.clone(),
14567                2,
14568                "alpha",
14569            )
14570            .unwrap();
14571        forwarding
14572            .complete_pending_relay(
14573                module_ctx.connection_id,
14574                pending.corr,
14575                RouteBindRelayOutcome::Accepted,
14576            )
14577            .unwrap();
14578        client_rx.try_recv().unwrap();
14579        for module_id in ["beta", "alpha"] {
14580            let replies = handler
14581                .handle_control_frame(
14582                    &module_ctx,
14583                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14584                )
14585                .await
14586                .unwrap();
14587            assert_eq!(replies.len(), 1, "second HELLO must be refused");
14588            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14589        }
14590        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14591        assert!(registry.get_module("beta").unwrap().is_none());
14592        assert!(matches!(
14593            forwarding
14594                .lookup_data_route(
14595                    client_ctx.connection_id,
14596                    pending.client_channel,
14597                    pending.client_epoch,
14598                )
14599                .unwrap(),
14600            DataRoute::Client(DataRouteState::Bound(_))
14601        ));
14602        assert!(handler
14603            .hello_launch_nonces
14604            .lock()
14605            .unwrap()
14606            .presented(module_ctx.connection_id, Some("alpha-nonce")));
14607        assert!(module_rx.try_recv().is_err());
14608    }
14609
14610    #[test]
14611    fn reserved_module_hello_requires_matching_launch_nonce() {
14612        let registry = Arc::new(Registry::default());
14613        let supervisor = SupervisorHandle::new();
14614        // The supervisor recorded the nonce it injected when it spawned the reserved
14615        // module; the HELLO verifier checks against the same shared handle.
14616        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14617        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14618
14619        // A HELLO with NO nonce is rejected.
14620        let no_nonce = handler
14621            .handle_control(
14622                ConnectionId::new(1),
14623                hello_frame("vault", PROTOCOL_VERSION, 1),
14624            )
14625            .unwrap();
14626        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14627        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14628        assert!(registry.get_module("vault").unwrap().is_none());
14629
14630        // A HELLO with the WRONG nonce is rejected.
14631        let wrong = handler
14632            .handle_control(
14633                ConnectionId::new(2),
14634                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14635            )
14636            .unwrap();
14637        assert_eq!(wrong[0].header.ty, FrameType::Error);
14638        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14639        assert!(registry.get_module("vault").unwrap().is_none());
14640
14641        // A HELLO with the CORRECT nonce registers.
14642        let ok = handler
14643            .handle_control(
14644                ConnectionId::new(3),
14645                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14646            )
14647            .unwrap();
14648        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14649        assert!(registry.get_module("vault").unwrap().is_some());
14650    }
14651
14652    #[test]
14653    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14654        let registry = Arc::new(Registry::default());
14655        let supervisor = SupervisorHandle::new();
14656        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14657        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14658        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14659
14660        let squat = handler
14661            .handle_control(
14662                ConnectionId::new(1),
14663                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14664            )
14665            .unwrap();
14666        assert_eq!(squat[0].header.ty, FrameType::Error);
14667        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14668        assert!(parse_error(&squat[0])["message"]
14669            .as_str()
14670            .unwrap()
14671            .contains("fed:"));
14672
14673        let accepted_peer = handler
14674            .handle_control(
14675                ConnectionId::new(2),
14676                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14677            )
14678            .unwrap();
14679        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14680
14681        let accepted_short = handler
14682            .handle_control(
14683                ConnectionId::new(3),
14684                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14685            )
14686            .unwrap();
14687        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14688
14689        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14690            let response = handler
14691                .handle_control(
14692                    ConnectionId::new(conn),
14693                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14694                )
14695                .unwrap();
14696            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14697        }
14698    }
14699
14700    #[test]
14701    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14702        let registry = Arc::new(Registry::default());
14703        let supervisor = SupervisorHandle::new();
14704        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14705        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14706        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14707        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14708
14709        let owner_nonce = handler
14710            .handle_control(
14711                ConnectionId::new(1),
14712                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14713            )
14714            .unwrap();
14715        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14716        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14717        assert!(registry.get_module("fed:special").unwrap().is_none());
14718
14719        let exact_nonce = handler
14720            .handle_control(
14721                ConnectionId::new(2),
14722                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14723            )
14724            .unwrap();
14725        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14726        assert!(registry.get_module("fed:special").unwrap().is_some());
14727    }
14728
14729    #[test]
14730    fn non_reserved_module_ignores_launch_nonce() {
14731        let registry = Arc::new(Registry::default());
14732        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14733        // registration succeeds whether a spawned process echoes a nonce or not.
14734        let handler = ControlHandler::new(Arc::clone(&registry));
14735        let no_nonce = handler
14736            .handle_control(
14737                ConnectionId::new(1),
14738                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14739            )
14740            .unwrap();
14741        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14742        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14743
14744        let echoed_nonce = handler
14745            .handle_control(
14746                ConnectionId::new(2),
14747                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14748            )
14749            .unwrap();
14750        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14751        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14752    }
14753
14754    #[test]
14755    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14756        let handler = ControlHandler::default();
14757        let conn = ConnectionId::new(1);
14758        let malformed = Frame::build(
14759            FrameType::Hello,
14760            control_flags(),
14761            0,
14762            0,
14763            3,
14764            b"{not json".to_vec(),
14765        )
14766        .unwrap();
14767
14768        let error = handler.handle_control(conn, malformed).unwrap();
14769        assert_eq!(error[0].header.ty, FrameType::Error);
14770        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14771
14772        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14773        let pong = handler.handle_control(conn, ping).unwrap();
14774        assert_eq!(pong[0].header.ty, FrameType::Pong);
14775        assert_eq!(pong[0].header.corr, 4);
14776    }
14777
14778    #[test]
14779    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14780        let registry = Arc::new(Registry::default());
14781        let handler = ControlHandler::new(Arc::clone(&registry));
14782
14783        handler
14784            .handle_control(
14785                ConnectionId::new(1),
14786                hello_frame("aft", PROTOCOL_VERSION, 1),
14787            )
14788            .unwrap();
14789        let duplicate = handler
14790            .handle_control(
14791                ConnectionId::new(2),
14792                hello_frame("aft", PROTOCOL_VERSION, 2),
14793            )
14794            .unwrap();
14795
14796        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14797        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14798        let registration = registry.get_module("aft").unwrap().unwrap();
14799        assert_eq!(registration.connection_id, ConnectionId::new(1));
14800    }
14801
14802    #[test]
14803    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14804        let registry = Arc::new(Registry::default());
14805        let forwarding = Arc::new(ForwardingTable::default());
14806        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14807        let handler =
14808            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14809                .with_process_liveness(process_liveness);
14810        let (ctx, route_channel, route_epoch) =
14811            bind_liveness_route(&registry, &forwarding, "aft-dead");
14812        let responses = handler
14813            .handle_route_poll(
14814                &ctx,
14815                route_poll_frame(41, PollKind::Liveness, route_channel),
14816                route_channel,
14817                route_epoch,
14818                PollKind::Liveness,
14819            )
14820            .unwrap();
14821
14822        assert_eq!(responses.len(), 1);
14823        assert_eq!(responses[0].header.ty, FrameType::Response);
14824        assert_route_poll_liveness(&responses[0], false);
14825    }
14826
14827    #[test]
14828    fn liveness_poll_without_process_source_uses_bound_route() {
14829        let registry = Arc::new(Registry::default());
14830        let forwarding = Arc::new(ForwardingTable::default());
14831        let handler =
14832            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14833        let (ctx, route_channel, route_epoch) =
14834            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14835        let responses = handler
14836            .handle_route_poll(
14837                &ctx,
14838                route_poll_frame(42, PollKind::Liveness, route_channel),
14839                route_channel,
14840                route_epoch,
14841                PollKind::Liveness,
14842            )
14843            .unwrap();
14844
14845        assert_route_poll_liveness(&responses[0], true);
14846    }
14847
14848    #[test]
14849    fn liveness_poll_untracked_process_source_uses_bound_route() {
14850        let registry = Arc::new(Registry::default());
14851        let forwarding = Arc::new(ForwardingTable::default());
14852        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14853        let handler =
14854            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14855                .with_process_liveness(process_liveness);
14856        let (ctx, route_channel, route_epoch) =
14857            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14858        let responses = handler
14859            .handle_route_poll(
14860                &ctx,
14861                route_poll_frame(43, PollKind::Liveness, route_channel),
14862                route_channel,
14863                route_epoch,
14864                PollKind::Liveness,
14865            )
14866            .unwrap();
14867
14868        assert_route_poll_liveness(&responses[0], true);
14869    }
14870
14871    #[tokio::test]
14872    async fn unknown_op_returns_unknown_control_op() {
14873        let handler = ControlHandler::default();
14874        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14875        let request = Frame::build(
14876            FrameType::Request,
14877            control_flags(),
14878            0,
14879            0,
14880            55,
14881            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14882        )
14883        .unwrap();
14884
14885        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14886
14887        assert_eq!(response.len(), 1);
14888        assert_eq!(response[0].header.ty, FrameType::Error);
14889        assert_eq!(response[0].header.corr, 55);
14890        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14891    }
14892
14893    #[tokio::test]
14894    async fn supervisor_provenance_rejects_unknown_exact_module() {
14895        let handler = ControlHandler::default();
14896        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14897        let request = Frame::build(
14898            FrameType::Request,
14899            control_flags(),
14900            0,
14901            0,
14902            57,
14903            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14904        )
14905        .unwrap();
14906
14907        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14908
14909        assert_eq!(response.len(), 1);
14910        assert_eq!(response[0].header.ty, FrameType::Error);
14911        assert_eq!(response[0].header.corr, 57);
14912        let error = parse_error(&response[0]);
14913        assert_eq!(error["code"], "unknown_module");
14914        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14915    }
14916
14917    #[test]
14918    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14919        let expected = subc_control::RunningImageAgreement::Unavailable {
14920            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14921        };
14922        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14923        assert_eq!(handler.provenance_probe_override, Some(expected));
14924    }
14925
14926    #[test]
14927    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14928        let verdict = reload_verdict(
14929            std::path::Path::new("/bin/new"),
14930            Some(std::path::Path::new("/bin/old")),
14931            subc_control::RunningImageAgreement::Unavailable {
14932                reason: subc_control::RunningImageUnavailableReason::HashFailed,
14933            },
14934        );
14935        assert!(matches!(
14936            verdict.path,
14937            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14938                if configured == std::path::Path::new("/bin/new")
14939                    && spawned_from == std::path::Path::new("/bin/old")
14940        ));
14941    }
14942
14943    #[test]
14944    fn launch_spec_without_running_digest_is_unavailable_not_match() {
14945        use subc_control::{
14946            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14947        };
14948        let configured = crate::supervise::ModuleSpec {
14949            module_id: "legacy".into(),
14950            program: "/module".into(),
14951            args: Vec::new(),
14952            env: Vec::new(),
14953            reserved: false,
14954            reserved_prefixes: Vec::new(),
14955            protocol: subc_control::ModuleProtocol::Subc,
14956            overlap: Default::default(),
14957        };
14958        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14959        assert_eq!(
14960            launch_spec_agreement(&configured, false, Some(&digest)),
14961            Agreement::Unavailable {
14962                reason: Reason::NotRunning
14963            }
14964        );
14965        assert_eq!(
14966            launch_spec_agreement(&configured, true, None),
14967            Agreement::Unavailable {
14968                reason: Reason::SpawnedSpecUnavailable
14969            }
14970        );
14971        assert_eq!(
14972            launch_spec_agreement(&configured, true, Some(&digest)),
14973            Agreement::Match
14974        );
14975    }
14976
14977    #[cfg(windows)]
14978    #[test]
14979    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14980        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14981        let configured = dir.join("module.exe");
14982        std::fs::write(&configured, b"executable path fixture").unwrap();
14983        let spawned = std::fs::canonicalize(&configured).unwrap();
14984        assert_ne!(
14985            configured, spawned,
14986            "fixture must exercise different path spellings"
14987        );
14988        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14989        let lower_case = configured.to_string_lossy().to_lowercase();
14990        for spelling in [
14991            &configured,
14992            Path::new(&forward_slashes),
14993            Path::new(&lower_case),
14994        ] {
14995            let verdict = reload_verdict(
14996                spelling,
14997                Some(&spawned),
14998                subc_control::RunningImageAgreement::Unavailable {
14999                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15000                },
15001            );
15002            assert_eq!(
15003                verdict.path,
15004                ReloadPathAgreement::Match,
15005                "configured {spelling:?}, spawned {spawned:?}"
15006            );
15007        }
15008    }
15009
15010    #[cfg(windows)]
15011    #[test]
15012    fn windows_reload_path_keeps_different_existing_program_pending() {
15013        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15014        let old = dir.join("old.exe");
15015        let new = dir.join("new.exe");
15016        std::fs::write(&old, b"same executable content").unwrap();
15017        std::fs::copy(&old, &new).unwrap();
15018        let spawned = std::fs::canonicalize(&old).unwrap();
15019        let image = subc_control::RunningImageAgreement::Unavailable {
15020            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15021        };
15022        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15023        assert!(
15024            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15025            if configured == new && spawned_from == spawned)
15026        );
15027        let missing = dir.join("missing.exe");
15028        assert!(matches!(
15029            reload_verdict(&missing, Some(&spawned), image).path,
15030            ReloadPathAgreement::Mismatch { .. }
15031        ));
15032    }
15033
15034    #[test]
15035    fn reload_verdict_detects_replaced_image_at_same_path() {
15036        let image = subc_control::RunningImageAgreement::Mismatch {
15037            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15038                digest: "old".into(),
15039            },
15040            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15041                digest: "new".into(),
15042            },
15043        };
15044        let verdict = reload_verdict(
15045            std::path::Path::new("/bin/same"),
15046            Some(std::path::Path::new("/bin/same")),
15047            image.clone(),
15048        );
15049        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15050        assert_eq!(verdict.image, image);
15051    }
15052
15053    #[test]
15054    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15055        let image = subc_control::RunningImageAgreement::Unavailable {
15056            reason: subc_control::RunningImageUnavailableReason::NotRunning,
15057        };
15058        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15059        assert_eq!(
15060            verdict.path,
15061            subc_control::ReloadPathAgreement::Unavailable {
15062                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15063            }
15064        );
15065        assert_eq!(verdict.image, image);
15066
15067        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15068            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15069        };
15070        let verdict = reload_verdict(
15071            std::path::Path::new("/bin/same"),
15072            Some(std::path::Path::new("/bin/same")),
15073            unconfirmed.clone(),
15074        );
15075        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15076        assert_eq!(verdict.image, unconfirmed);
15077    }
15078
15079    #[test]
15080    fn reload_verdict_preserves_each_image_unavailability_reason() {
15081        use subc_control::RunningImageUnavailableReason as Reason;
15082
15083        for reason in [
15084            Reason::NotRunning,
15085            Reason::UnsupportedPlatform,
15086            Reason::RunningExecutableUnreadable,
15087            Reason::SpawnedPathUnreadable,
15088            Reason::HashFailed,
15089            Reason::ProcessIdentityUnconfirmed,
15090            Reason::Unknown("future_probe_reason".to_string()),
15091        ] {
15092            let image = subc_control::RunningImageAgreement::Unavailable {
15093                reason: reason.clone(),
15094            };
15095            let verdict = reload_verdict(
15096                std::path::Path::new("/bin/same"),
15097                Some(std::path::Path::new("/bin/same")),
15098                image.clone(),
15099            );
15100            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15101            assert_eq!(verdict.image, image, "{reason:?}");
15102        }
15103    }
15104
15105    #[tokio::test]
15106    async fn malformed_control_bodies_return_invalid_control_body() {
15107        let handler = ControlHandler::default();
15108        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15109
15110        for (corr, body) in [
15111            (56, br#"{"route_channel":1}"#.as_slice()),
15112            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15113            (
15114                58,
15115                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15116            ),
15117        ] {
15118            let request = Frame::build(
15119                FrameType::Request,
15120                control_flags(),
15121                0,
15122                0,
15123                corr,
15124                body.to_vec(),
15125            )
15126            .unwrap();
15127            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15128
15129            assert_eq!(response.len(), 1);
15130            assert_eq!(response[0].header.ty, FrameType::Error);
15131            assert_eq!(response[0].header.corr, corr);
15132            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15133        }
15134    }
15135
15136    #[tokio::test]
15137    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15138        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15139        let registry = Arc::new(Registry::default());
15140        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15141        let router = Router::with_control_handler(Arc::clone(&control));
15142        let connection = router.begin_connection();
15143        let (ctx, mut rx) = route_ctx(connection.id());
15144
15145        router
15146            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15147            .await
15148            .unwrap();
15149        let response = rx.recv().await.unwrap();
15150        let ack = parse_ack(&response);
15151        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15152        let channel = 1;
15153
15154        let goodbye =
15155            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15156        router.route_for_connection(&ctx, goodbye).await.unwrap();
15157        assert!(rx.try_recv().is_err());
15158        assert!(registry.get_module("aft").unwrap().is_none());
15159
15160        router
15161            .route_for_connection(&ctx, channel_request(channel, 13))
15162            .await
15163            .unwrap();
15164        let error_frame = rx.recv().await.unwrap();
15165        assert_eq!(error_frame.header.ty, FrameType::Error);
15166        assert_eq!(error_frame.header.channel, channel);
15167        let captured = crate::router::test_log::captured_logs(&logs);
15168        assert_eq!(
15169            captured
15170                .lines()
15171                .filter(|line| {
15172                    line.contains("module_id=aft")
15173                        && line.contains("reason=explicit_goodbye")
15174                        && line.contains("module registration ended")
15175                })
15176                .count(),
15177            1,
15178            "unexpected GOODBYE registry log: {captured}"
15179        );
15180    }
15181
15182    #[tokio::test]
15183    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15184        let registry = Arc::new(Registry::default());
15185        let handler = ControlHandler::new(registry).with_capability_config(
15186            [("prov".to_string(), true), ("cons".to_string(), true)],
15187            BTreeMap::new(),
15188        );
15189        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15190        register_capability_manifest(
15191            &handler,
15192            &provider_ctx,
15193            &mut provider_rx,
15194            capability_manifest("prov", &["thing/v1"], &[]),
15195            1,
15196        )
15197        .await;
15198        let mut consumer = capability_manifest("cons", &[], &[]);
15199        consumer.capabilities.as_mut().unwrap().requires.push(
15200            subc_protocol::manifest::CapabilityRequirement {
15201                capability: "thing/v1".to_string(),
15202                need: subc_protocol::manifest::CapabilityNeed::Required,
15203            },
15204        );
15205        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15206        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15207        assert_eq!(
15208            handler.capability_evaluator.verdict("cons", "thing/v1"),
15209            Some(CapabilityVerdict::Provided)
15210        );
15211        let (mut client_rx, _) = open_route_for_capability_test(
15212            &handler,
15213            &provider_ctx,
15214            &mut provider_rx,
15215            703,
15216            3,
15217            "prov",
15218            None,
15219        )
15220        .await;
15221        let goodbye =
15222            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15223        handler
15224            .handle_control_frame(&provider_ctx, goodbye)
15225            .await
15226            .unwrap();
15227        assert_eq!(
15228            handler.capability_evaluator.verdict("cons", "thing/v1"),
15229            Some(CapabilityVerdict::NeverProvided)
15230        );
15231        let closed = client_rx
15232            .try_recv()
15233            .expect("GOODBYE pushes route.closed before route GOODBYE");
15234        assert!(
15235            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15236            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15237        );
15238        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15239        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15240    }
15241
15242    struct RouteClosedFixture {
15243        handler: ControlHandler,
15244        #[cfg(unix)]
15245        supervisor: Supervisor,
15246        module_ctx: RouteCtx,
15247        #[cfg(unix)]
15248        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15249        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15250        route: Arc<crate::forwarding::RouteBinding>,
15251    }
15252
15253    async fn route_closed_fixture() -> RouteClosedFixture {
15254        let registry = Arc::new(Registry::default());
15255        let handler = ControlHandler::new(Arc::clone(&registry));
15256        #[cfg(unix)]
15257        let supervisor = Supervisor::new(registry, RestartPolicy::default())
15258            .with_forwarding(Arc::clone(&handler.forwarding));
15259        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15260        register_capability_manifest(
15261            &handler,
15262            &module_ctx,
15263            &mut module_rx,
15264            capability_manifest("prov", &[], &[]),
15265            1,
15266        )
15267        .await;
15268        let (client_rx, bind) = open_route_for_capability_test(
15269            &handler,
15270            &module_ctx,
15271            &mut module_rx,
15272            702,
15273            2,
15274            "prov",
15275            None,
15276        )
15277        .await;
15278        let ModuleControlRequest::RouteBind {
15279            route_channel: channel,
15280            epoch: route_epoch,
15281            ..
15282        } = bind
15283        else {
15284            panic!("expected route.bind");
15285        };
15286        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15287            handler
15288                .forwarding
15289                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15290                .unwrap()
15291        else {
15292            panic!("the acknowledged route must be bound");
15293        };
15294        RouteClosedFixture {
15295            handler,
15296            #[cfg(unix)]
15297            supervisor,
15298            module_ctx,
15299            #[cfg(unix)]
15300            module_rx,
15301            client_rx,
15302            route,
15303        }
15304    }
15305
15306    impl RouteClosedFixture {
15307        #[cfg(unix)]
15308        async fn cut(&mut self) {
15309            let notices = async {
15310                let mut draining = self.module_rx.recv().await.unwrap();
15311                assert!(matches!(
15312                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15313                        &draining.body
15314                    )
15315                    .unwrap(),
15316                    subc_protocol::session::ModuleControlCommand::Draining {
15317                        reason: RouteCloseReason::Restart,
15318                        ..
15319                    }
15320                ));
15321                draining.flushed.take().unwrap().send(()).unwrap();
15322                let mut closing = self.client_rx.recv().await.unwrap();
15323                assert!(matches!(
15324                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15325                    ClientControlPush::RouteClosing {
15326                        reason: RouteCloseReason::Restart,
15327                        ..
15328                    }
15329                ));
15330                closing.flushed.take().unwrap().send(()).unwrap();
15331            };
15332            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15333            result.unwrap();
15334        }
15335
15336        fn disconnect(&mut self) -> ClientControlPush {
15337            self.handler
15338                .cleanup_connection(self.module_ctx.connection_id)
15339                .unwrap();
15340            let closed = self.client_rx.try_recv().unwrap();
15341            assert_eq!(closed.header.ty, FrameType::Push);
15342            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15343            assert!(matches!(
15344                &push,
15345                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15346                    if channels == &[self.route.client_channel]
15347            ));
15348            assert_eq!(
15349                self.client_rx.try_recv().unwrap().header.ty,
15350                FrameType::Goodbye
15351            );
15352            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15353            push
15354        }
15355    }
15356
15357    #[cfg(unix)]
15358    #[tokio::test(start_paused = true)]
15359    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15360        let mut fixture = route_closed_fixture().await;
15361        fixture.cut().await;
15362        assert!(matches!(
15363            fixture.disconnect(),
15364            ClientControlPush::RouteClosed {
15365                reason: RouteCloseReason::Restart,
15366                drained: true,
15367                ..
15368            }
15369        ));
15370    }
15371
15372    #[cfg(unix)]
15373    #[tokio::test(start_paused = true)]
15374    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15375        let mut fixture = route_closed_fixture().await;
15376        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15377        let started = tokio::time::Instant::now();
15378        fixture.cut().await;
15379        assert!(started.elapsed() >= Duration::from_secs(2));
15380        assert_eq!(fixture.route.flow.in_flight(), 1);
15381        assert!(matches!(
15382            fixture.disconnect(),
15383            ClientControlPush::RouteClosed {
15384                reason: RouteCloseReason::Restart,
15385                drained: false,
15386                ..
15387            }
15388        ));
15389    }
15390
15391    #[tokio::test]
15392    async fn module_connection_crash_reports_crash_and_not_drained() {
15393        let mut fixture = route_closed_fixture().await;
15394        // The server calls cleanup_connection on module EOF. Without a planned
15395        // daemon shutdown, that disconnect must remain a crash with no prior
15396        // route.closing notice, even when the route has no in-flight calls.
15397        assert!(matches!(
15398            fixture.disconnect(),
15399            ClientControlPush::RouteClosed {
15400                reason: RouteCloseReason::Crash,
15401                drained: false,
15402                ..
15403            }
15404        ));
15405    }
15406
15407    #[tokio::test]
15408    async fn dropping_router_connection_releases_registration() {
15409        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15410        let registry = Arc::new(Registry::default());
15411        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15412        let router = Router::with_control_handler(Arc::clone(&control));
15413        let connection = router.begin_connection();
15414        let connection_id = connection.id();
15415        let (ctx, mut rx) = route_ctx(connection_id);
15416
15417        router
15418            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15419            .await
15420            .unwrap();
15421        let response = rx.recv().await.unwrap();
15422        let ack = parse_ack(&response);
15423        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15424        assert!(registry.get_module("aft").unwrap().is_some());
15425
15426        drop(connection);
15427
15428        assert!(registry.get_module("aft").unwrap().is_none());
15429        assert_eq!(registry.active_registration_count().unwrap(), 0);
15430
15431        control
15432            .cleanup_connection(ConnectionId::new(u64::MAX))
15433            .unwrap();
15434        let captured = crate::router::test_log::captured_logs(&logs);
15435        println!("captured registry lifecycle logs:\n{captured}");
15436        let events: Vec<_> = captured
15437            .lines()
15438            .filter(|line| {
15439                line.contains("module registered module_id=aft ")
15440                    || line.contains("module registration ended")
15441            })
15442            .collect();
15443        assert_eq!(
15444            events.len(),
15445            2,
15446            "unexpected registry lifecycle logs: {captured}"
15447        );
15448        assert!(events[0].contains("module registered module_id=aft "));
15449        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15450        assert!(events[1].contains(&format!(
15451            "module_id=aft connection_id={}",
15452            connection_id.get()
15453        )));
15454        assert!(events[1].contains("reason=connection_closed"));
15455        assert!(events[1].contains("module registration ended"));
15456        assert_eq!(
15457            captured
15458                .lines()
15459                .filter(|line| line.contains("module registered module_id=aft "))
15460                .count(),
15461            1,
15462            "legacy registration admission line must appear once: {captured}"
15463        );
15464    }
15465
15466    #[tokio::test]
15467    async fn hello_registration_keeps_legacy_module_registered_line_once() {
15468        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15469        let registry = Arc::new(Registry::default());
15470        let control = ControlHandler::new(Arc::clone(&registry));
15471        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15472        hello_via_sink(
15473            &control,
15474            &ctx,
15475            &mut rx,
15476            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15477        )
15478        .await;
15479
15480        let captured = crate::router::test_log::captured_logs(&logs);
15481        let admissions: Vec<_> = captured
15482            .lines()
15483            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15484            .collect();
15485        assert_eq!(
15486            admissions.len(),
15487            1,
15488            "legacy admission line must remain exactly once: {captured}"
15489        );
15490        assert!(admissions[0].contains("routable_provider=true"));
15491        assert!(admissions[0].contains("connection_id=777"));
15492    }
15493
15494    fn capability_manifest(
15495        module_id: &str,
15496        provides: &[&str],
15497        must_never_reach: &[&str],
15498    ) -> ModuleManifest {
15499        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15500        manifest.capabilities = Some(CapabilityDeclarations {
15501            provides: provides
15502                .iter()
15503                .map(|capability| (*capability).to_string())
15504                .collect(),
15505            requires: Vec::new(),
15506            must_never_reach: must_never_reach
15507                .iter()
15508                .map(|capability| (*capability).to_string())
15509                .collect(),
15510        });
15511        manifest
15512    }
15513
15514    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15515        Frame::build(
15516            FrameType::Hello,
15517            control_flags(),
15518            0,
15519            0,
15520            corr,
15521            serde_json::to_vec(&ModuleHelloBody {
15522                protocol_ver: manifest.protocol_ver,
15523                manifest,
15524                control_ops: None,
15525                launch_nonce: None,
15526            })
15527            .expect("capability test HELLO serializes"),
15528        )
15529        .expect("capability test HELLO frame builds")
15530    }
15531
15532    fn catalog_update_with_capabilities_frame(
15533        corr: u64,
15534        capabilities: CapabilityDeclarations,
15535    ) -> Frame {
15536        Frame::build(
15537            FrameType::Request,
15538            control_flags(),
15539            0,
15540            0,
15541            corr,
15542            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15543                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15544                capabilities: Some(capabilities),
15545                ready: None,
15546            })
15547            .expect("capability catalog.update serializes"),
15548        )
15549        .expect("capability catalog.update frame builds")
15550    }
15551
15552    async fn register_capability_manifest(
15553        handler: &ControlHandler,
15554        ctx: &RouteCtx,
15555        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15556        manifest: ModuleManifest,
15557        corr: u64,
15558    ) {
15559        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15560    }
15561
15562    async fn open_route_for_capability_test(
15563        handler: &ControlHandler,
15564        target_ctx: &RouteCtx,
15565        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15566        client_connection_id: u64,
15567        corr: u64,
15568        target_module_id: &str,
15569        consumer_identity: Option<ConsumerIdentity>,
15570    ) -> (
15571        mpsc::Receiver<crate::router::OutboundFrame>,
15572        ModuleControlRequest,
15573    ) {
15574        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15575        let route_handler = handler.clone();
15576        let target_module_id = target_module_id.to_string();
15577        let route_task = tokio::spawn(async move {
15578            route_handler
15579                .handle_control_frame(
15580                    &client_ctx,
15581                    route_open_frame_with_admission_facts(
15582                        corr,
15583                        &target_module_id,
15584                        unique_project_root("admission-facts"),
15585                        consumer_identity,
15586                        None,
15587                    ),
15588                )
15589                .await
15590                .expect("capability test route.open succeeds")
15591        });
15592        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15593            .await
15594            .expect("capability test route.open must reach route.bind")
15595            .expect("target control receiver stays open");
15596        let bind_request: ModuleControlRequest =
15597            serde_json::from_slice(&bind.body).expect("route.bind decodes");
15598        handler
15599            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15600            .await
15601            .expect("capability test route.bind ACK succeeds");
15602        assert!(route_task.await.expect("route.open task joins").is_empty());
15603        let opened = client_rx
15604            .recv()
15605            .await
15606            .expect("successful route.open publishes a response");
15607        assert!(matches!(
15608            serde_json::from_slice::<ClientControlResponse>(&opened.body),
15609            Ok(ClientControlResponse::RouteOpen { .. })
15610        ));
15611        (client_rx, bind_request)
15612    }
15613
15614    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15615        assert_eq!(frame.header.ty, FrameType::Push);
15616        assert_eq!(frame.header.channel, 0);
15617        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15618            .expect("route.closed control push decodes");
15619        let ClientControlPush::RouteClosed { channels, .. } = &push else {
15620            panic!("expected route.closed");
15621        };
15622        assert_eq!(channels.len(), 1, "exactly one violating route closed");
15623        let closed_channel = channels[0];
15624        let channels = channels.clone();
15625        assert_eq!(
15626            push,
15627            ClientControlPush::RouteClosed {
15628                module_id: target_module_id.to_string(),
15629                channels,
15630                reason: RouteCloseReason::CapabilityDenied,
15631                drained: false,
15632                abandoned: 0,
15633                excluded_subscriptions: 0,
15634                terminal: Some(false),
15635            }
15636        );
15637        closed_channel
15638    }
15639
15640    #[tokio::test]
15641    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15642        let registry = Arc::new(Registry::default());
15643        let forwarding = Arc::new(ForwardingTable::default());
15644        let supervisor = SupervisorHandle::new();
15645        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15646        let handler =
15647            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15648                .with_supervisor(supervisor);
15649        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15650        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15651        register_capability_manifest(
15652            &handler,
15653            &target_ctx,
15654            &mut target_rx,
15655            capability_manifest("target", &["credentials-provider/v1"], &[]),
15656            1,
15657        )
15658        .await;
15659        register_capability_manifest(
15660            &handler,
15661            &opener_ctx,
15662            &mut opener_rx,
15663            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15664            2,
15665        )
15666        .await;
15667
15668        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15669        let replies = handler
15670            .handle_control_frame(
15671                &client_ctx,
15672                route_open_frame_with_admission_facts(
15673                    3,
15674                    "target",
15675                    unique_project_root("admission-facts"),
15676                    Some(ConsumerIdentity {
15677                        module_id: "opener".to_string(),
15678                        launch_nonce: "opener-nonce".to_string(),
15679                    }),
15680                    None,
15681                ),
15682            )
15683            .await
15684            .expect("denied route.open returns a typed frame");
15685        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15686        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15687        assert!(
15688            target_rx.try_recv().is_err(),
15689            "forbidden route.open must not relay route.bind"
15690        );
15691    }
15692
15693    #[tokio::test]
15694    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15695        let registry = Arc::new(Registry::default());
15696        let forwarding = Arc::new(ForwardingTable::default());
15697        let supervisor = SupervisorHandle::new();
15698        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15699        let handler =
15700            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15701                .with_supervisor(supervisor);
15702        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15703        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15704        register_capability_manifest(
15705            &handler,
15706            &target_ctx,
15707            &mut target_rx,
15708            capability_manifest("target", &["credentials-provider/v1"], &[]),
15709            1,
15710        )
15711        .await;
15712        register_capability_manifest(
15713            &handler,
15714            &old_opener_ctx,
15715            &mut old_opener_rx,
15716            capability_manifest("opener", &[], &[]),
15717            2,
15718        )
15719        .await;
15720        let (mut client_rx, _) = open_route_for_capability_test(
15721            &handler,
15722            &target_ctx,
15723            &mut target_rx,
15724            712,
15725            3,
15726            "target",
15727            Some(ConsumerIdentity {
15728                module_id: "opener".to_string(),
15729                launch_nonce: "opener-nonce".to_string(),
15730            }),
15731        )
15732        .await;
15733        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15734
15735        handler
15736            .cleanup_connection(old_opener_ctx.connection_id)
15737            .expect("old opener registration cleans up");
15738        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15739        register_capability_manifest(
15740            &handler,
15741            &new_opener_ctx,
15742            &mut new_opener_rx,
15743            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15744            4,
15745        )
15746        .await;
15747
15748        let closed_channel = assert_capability_denied_push(
15749            client_rx
15750                .try_recv()
15751                .expect("HELLO deny addition must emit route.closed")
15752                .frame,
15753            "target",
15754        );
15755        let goodbye = client_rx
15756            .try_recv()
15757            .expect("HELLO deny addition must send the client a route GOODBYE")
15758            .frame;
15759        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15760        assert_eq!(goodbye.header.channel, closed_channel);
15761        assert_ne!(goodbye.header.channel, 0);
15762        assert_ne!(goodbye.header.epoch, 0);
15763        assert!(client_rx.try_recv().is_err());
15764        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15765        assert!(matches!(
15766            target_rx.try_recv(),
15767            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15768        ));
15769    }
15770
15771    #[tokio::test]
15772    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15773        let registry = Arc::new(Registry::default());
15774        let forwarding = Arc::new(ForwardingTable::default());
15775        let supervisor = SupervisorHandle::new();
15776        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15777        let handler =
15778            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15779                .with_supervisor(supervisor);
15780        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15781        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15782        register_capability_manifest(
15783            &handler,
15784            &target_ctx,
15785            &mut target_rx,
15786            capability_manifest("target", &[], &[]),
15787            1,
15788        )
15789        .await;
15790        register_capability_manifest(
15791            &handler,
15792            &opener_ctx,
15793            &mut opener_rx,
15794            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15795            2,
15796        )
15797        .await;
15798        let (mut client_rx, _) = open_route_for_capability_test(
15799            &handler,
15800            &target_ctx,
15801            &mut target_rx,
15802            722,
15803            3,
15804            "target",
15805            Some(ConsumerIdentity {
15806                module_id: "opener".to_string(),
15807                launch_nonce: "opener-nonce".to_string(),
15808            }),
15809        )
15810        .await;
15811        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15812
15813        let replies = handler
15814            .handle_control_frame(
15815                &target_ctx,
15816                catalog_update_with_capabilities_frame(
15817                    4,
15818                    CapabilityDeclarations {
15819                        provides: vec!["credentials-provider/v1".to_string()],
15820                        requires: Vec::new(),
15821                        must_never_reach: Vec::new(),
15822                    },
15823                ),
15824            )
15825            .await
15826            .expect("claim catalog.update succeeds");
15827        assert!(matches!(
15828            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15829            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15830        ));
15831        let closed_channel = assert_capability_denied_push(
15832            client_rx
15833                .try_recv()
15834                .expect("claim addition must emit route.closed")
15835                .frame,
15836            "target",
15837        );
15838        let goodbye = client_rx
15839            .try_recv()
15840            .expect("claim addition must send the client a route GOODBYE")
15841            .frame;
15842        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15843        assert_eq!(goodbye.header.channel, closed_channel);
15844        assert_ne!(goodbye.header.channel, 0);
15845        assert_ne!(goodbye.header.epoch, 0);
15846        assert!(client_rx.try_recv().is_err());
15847        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15848        assert!(matches!(
15849            target_rx.try_recv(),
15850            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15851        ));
15852    }
15853
15854    #[tokio::test]
15855    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15856        let registry = Arc::new(Registry::default());
15857        let forwarding = Arc::new(ForwardingTable::default());
15858        let supervisor = SupervisorHandle::new();
15859        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15860        let handler =
15861            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15862                .with_supervisor(supervisor);
15863        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15864        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15865        register_capability_manifest(
15866            &handler,
15867            &target_ctx,
15868            &mut target_rx,
15869            capability_manifest("target", &["credentials-provider/v1"], &[]),
15870            1,
15871        )
15872        .await;
15873        register_capability_manifest(
15874            &handler,
15875            &opener_ctx,
15876            &mut opener_rx,
15877            capability_manifest("opener", &[], &[]),
15878            2,
15879        )
15880        .await;
15881        let (mut client_rx, _) = open_route_for_capability_test(
15882            &handler,
15883            &target_ctx,
15884            &mut target_rx,
15885            732,
15886            3,
15887            "target",
15888            Some(ConsumerIdentity {
15889                module_id: "opener".to_string(),
15890                launch_nonce: "opener-nonce".to_string(),
15891            }),
15892        )
15893        .await;
15894        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15895
15896        handler
15897            .handle_control_frame(
15898                &target_ctx,
15899                catalog_update_with_capabilities_frame(
15900                    4,
15901                    CapabilityDeclarations {
15902                        provides: Vec::new(),
15903                        requires: Vec::new(),
15904                        must_never_reach: Vec::new(),
15905                    },
15906                ),
15907            )
15908            .await
15909            .expect("claim removal catalog.update succeeds");
15910        assert_eq!(
15911            forwarding.active_binding_count().unwrap(),
15912            1,
15913            "removing an attested target claim must leave the route census unchanged"
15914        );
15915        assert!(
15916            client_rx.try_recv().is_err(),
15917            "claim removal must not emit route.closed or a client route GOODBYE"
15918        );
15919        assert!(
15920            target_rx.try_recv().is_err(),
15921            "claim removal must not send the target a route GOODBYE"
15922        );
15923    }
15924
15925    /// A direct client may open a route to a denied capability provider; this
15926    /// policy applies only to attested supervised module origins, not to direct clients.
15927    #[tokio::test]
15928    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15929        let registry = Arc::new(Registry::default());
15930        let forwarding = Arc::new(ForwardingTable::default());
15931        let supervisor = SupervisorHandle::new();
15932        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15933        let handler =
15934            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15935                .with_supervisor(supervisor);
15936        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15937        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15938        register_capability_manifest(
15939            &handler,
15940            &target_ctx,
15941            &mut target_rx,
15942            capability_manifest("target", &["credentials-provider/v1"], &[]),
15943            1,
15944        )
15945        .await;
15946        register_capability_manifest(
15947            &handler,
15948            &opener_ctx,
15949            &mut opener_rx,
15950            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15951            2,
15952        )
15953        .await;
15954
15955        let (_client_rx, bind) = open_route_for_capability_test(
15956            &handler,
15957            &target_ctx,
15958            &mut target_rx,
15959            742,
15960            3,
15961            "target",
15962            None,
15963        )
15964        .await;
15965        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15966            panic!("direct scope-honesty route must bind");
15967        };
15968        assert_eq!(principal, Some(Principal::Direct));
15969        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15970    }
15971
15972    /// A module that denies a capability receives no self-route exemption when it
15973    /// also attestedly provides that capability.
15974    #[tokio::test]
15975    async fn must_never_reach_self_route_is_capability_forbidden() {
15976        let registry = Arc::new(Registry::default());
15977        let forwarding = Arc::new(ForwardingTable::default());
15978        let supervisor = SupervisorHandle::new();
15979        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15980        let handler =
15981            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15982                .with_supervisor(supervisor);
15983        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15984        register_capability_manifest(
15985            &handler,
15986            &self_ctx,
15987            &mut self_rx,
15988            capability_manifest(
15989                "self-provider",
15990                &["credentials-provider/v1"],
15991                &["credentials-provider/v1"],
15992            ),
15993            1,
15994        )
15995        .await;
15996
15997        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15998        let replies = handler
15999            .handle_control_frame(
16000                &client_ctx,
16001                route_open_frame_with_admission_facts(
16002                    2,
16003                    "self-provider",
16004                    unique_project_root("admission-facts"),
16005                    Some(ConsumerIdentity {
16006                        module_id: "self-provider".to_string(),
16007                        launch_nonce: "self-nonce".to_string(),
16008                    }),
16009                    None,
16010                ),
16011            )
16012            .await
16013            .expect("self-route refusal returns a typed frame");
16014        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16015        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16016        assert!(
16017            self_rx.try_recv().is_err(),
16018            "self denial must not relay route.bind"
16019        );
16020    }
16021
16022    #[test]
16023    fn unsupported_channel_zero_frame_returns_error() {
16024        let handler = ControlHandler::default();
16025        let request = Frame::build(
16026            FrameType::Request,
16027            control_flags(),
16028            0,
16029            0,
16030            21,
16031            b"opaque".to_vec(),
16032        )
16033        .unwrap();
16034
16035        let response = handler
16036            .handle_control(ConnectionId::new(1), request)
16037            .unwrap();
16038
16039        assert_eq!(response[0].header.ty, FrameType::Error);
16040        assert_eq!(
16041            parse_error(&response[0])["code"],
16042            "unsupported_control_frame"
16043        );
16044    }
16045
16046    /// Blue/green swap at the control-plane boundary. The supervisor that opens
16047    /// a swap is not wired yet, so the candidate is registered here directly
16048    /// into the registry and forwarding candidate slots, the way the swap's
16049    /// HELLO admission will.
16050    mod swap {
16051        use super::*;
16052
16053        const INCUMBENT: ConnectionId = ConnectionId::new(30);
16054        const CANDIDATE: ConnectionId = ConnectionId::new(40);
16055
16056        struct Swap {
16057            registry: Arc<Registry>,
16058            forwarding: Arc<ForwardingTable>,
16059            handler: ControlHandler,
16060            incumbent_ctx: RouteCtx,
16061            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16062            candidate_ctx: RouteCtx,
16063            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16064        }
16065
16066        async fn swap_with_incumbent() -> Swap {
16067            let registry = Arc::new(Registry::default());
16068            let forwarding = Arc::new(ForwardingTable::default());
16069            let handler =
16070                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
16071            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16072            hello_via_sink(
16073                &handler,
16074                &incumbent_ctx,
16075                &mut incumbent_rx,
16076                hello_frame("aft", PROTOCOL_VERSION, 7),
16077            )
16078            .await;
16079            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16080            Swap {
16081                registry,
16082                forwarding,
16083                handler,
16084                incumbent_ctx,
16085                incumbent_rx,
16086                candidate_ctx,
16087                candidate_rx,
16088            }
16089        }
16090
16091        fn register_candidate(swap: &Swap, ready: Option<bool>) {
16092            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16093            candidate_manifest.ready = ready;
16094            let registration = swap
16095                .registry
16096                .register_candidate_with_control_ops(
16097                    candidate_manifest,
16098                    PROTOCOL_VERSION,
16099                    CANDIDATE,
16100                    module_baseline_control_ops(),
16101                )
16102                .unwrap();
16103            swap.forwarding
16104                .register_candidate_module_connection(
16105                    CANDIDATE,
16106                    "aft".to_string(),
16107                    PROTOCOL_VERSION,
16108                    manifest_concurrency(&registration.manifest),
16109                    swap.candidate_ctx.egress.clone(),
16110                )
16111                .unwrap();
16112        }
16113
16114        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16115            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16116            swap.registry.promote_candidate("aft").unwrap().unwrap();
16117            cutover.incumbent.unwrap()
16118        }
16119
16120        /// A swap candidate's HELLO reserves its candidate registry entry
16121        /// before installing its candidate forwarding endpoint. The swap
16122        /// supervisor promotes a ready candidate as soon as it reads it as
16123        /// registered, and promotion cuts forwarding over to the candidate
16124        /// endpoint, so the lookup the supervisor reads
16125        /// (`Registry::published_registration`) must not return the candidate
16126        /// until that endpoint exists. Otherwise the cutover finds no candidate
16127        /// endpoint and the swap fails as if the candidate had disconnected.
16128        #[tokio::test]
16129        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16130            let mut swap = swap_with_incumbent().await;
16131            let hold = Arc::new(std::sync::Barrier::new(2));
16132            let handler = swap
16133                .handler
16134                .clone()
16135                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16136            handler
16137                .supervisor
16138                .open_swap("aft", "candidate-nonce".to_string());
16139            let hello = {
16140                let handler = handler.clone();
16141                let candidate_ctx = swap.candidate_ctx.clone();
16142                std::thread::spawn(move || {
16143                    handler
16144                        .handle_hello(
16145                            candidate_ctx.connection_id,
16146                            Some(candidate_ctx.egress.clone()),
16147                            hello_frame_with_nonce(
16148                                "aft",
16149                                PROTOCOL_VERSION,
16150                                8,
16151                                Some("candidate-nonce"),
16152                            ),
16153                        )
16154                        .unwrap()
16155                })
16156            };
16157
16158            hold.wait();
16159            // Parked: the candidate slot is claimed but not published, and
16160            // forwarding has no candidate endpoint.
16161            assert!(swap
16162                .registry
16163                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16164                .unwrap()
16165                .is_some());
16166            assert!(swap
16167                .registry
16168                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16169                .unwrap()
16170                .is_none());
16171            assert!(swap
16172                .forwarding
16173                .module_endpoint_for_connection(CANDIDATE)
16174                .unwrap()
16175                .is_none());
16176
16177            hold.wait();
16178            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16179            assert_eq!(
16180                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16181                FrameType::HelloAck
16182            );
16183            let published = swap
16184                .registry
16185                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16186                .unwrap()
16187                .expect("candidate is published once its HELLO completes");
16188            assert_eq!(published.connection_id, CANDIDATE);
16189            assert!(swap
16190                .forwarding
16191                .module_endpoint_for_connection(CANDIDATE)
16192                .unwrap()
16193                .is_some());
16194        }
16195
16196        fn keyed_total(counters: &Value, key: &str) -> u64 {
16197            counters[key]
16198                .as_object()
16199                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16200                .unwrap_or(0)
16201        }
16202
16203        #[tokio::test]
16204        async fn replacement_logs_old_end_and_new_admission_once() {
16205            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16206            let mut swap = swap_with_incumbent().await;
16207            swap.handler
16208                .supervisor
16209                .open_swap("aft", "candidate-nonce".to_string());
16210            hello_via_sink(
16211                &swap.handler,
16212                &swap.candidate_ctx,
16213                &mut swap.candidate_rx,
16214                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16215            )
16216            .await;
16217            cutover(&swap);
16218            swap.handler.cleanup_connection(INCUMBENT).unwrap();
16219
16220            let captured = crate::router::test_log::captured_logs(&logs);
16221            println!("captured replacement registry logs:\n{captured}");
16222            let new_admissions: Vec<_> = captured
16223                .lines()
16224                .filter(|line| {
16225                    line.contains("connection_id=40")
16226                        && line.contains("swap candidate registered; not routable until cutover")
16227                })
16228                .collect();
16229            assert_eq!(
16230                new_admissions.len(),
16231                1,
16232                "unexpected admission logs: {captured}"
16233            );
16234            assert!(new_admissions[0].contains("module_id=aft"));
16235            assert!(new_admissions[0].contains("connection_id=40"));
16236            assert!(
16237                new_admissions[0].contains("swap candidate registered; not routable until cutover")
16238            );
16239            assert!(new_admissions[0].contains("ready=true"));
16240
16241            let old_admissions: Vec<_> = captured
16242                .lines()
16243                .filter(|line| {
16244                    line.contains("module registered module_id=aft ")
16245                        && line.contains("connection_id=30")
16246                })
16247                .collect();
16248            assert_eq!(
16249                old_admissions.len(),
16250                1,
16251                "unexpected incumbent admission: {captured}"
16252            );
16253
16254            let old_ends: Vec<_> = captured
16255                .lines()
16256                .filter(|line| {
16257                    line.contains("connection_id=30") && line.contains("module registration ended")
16258                })
16259                .collect();
16260            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16261            assert!(old_ends[0].contains("module_id=aft"));
16262            assert!(old_ends[0].contains("connection_id=30"));
16263            assert!(old_ends[0].contains("reason=replaced"));
16264            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16265            assert_eq!(
16266                captured
16267                    .lines()
16268                    .filter(|line| line.contains("module registration promoted"))
16269                    .count(),
16270                1,
16271                "unexpected promotion logs: {captured}"
16272            );
16273        }
16274
16275        /// An ack from the incumbent for a bind it was sent before cutover,
16276        /// arriving before the incumbent is drained. The incumbent is the live
16277        /// connection carrying every other client's routes, so the ack must
16278        /// not end it: the waiting client is told to retry, the reservation is
16279        /// given back, and the incumbent is told to drop just that binding.
16280        #[tokio::test]
16281        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16282            let mut swap = swap_with_incumbent().await;
16283            let handler = swap.handler.clone();
16284
16285            // A co-tenant route, bound on the incumbent before the swap.
16286            let cotenant = ConnectionId::new(31);
16287            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16288            let (cotenant_task, cotenant_bind) = relay_route_open(
16289                &handler,
16290                cotenant,
16291                &cotenant_ctx.egress,
16292                &mut swap.incumbent_rx,
16293                100,
16294                "aft",
16295                "swap-cotenant",
16296            )
16297            .await;
16298            handler
16299                .handle_control_frame(
16300                    &swap.incumbent_ctx,
16301                    route_bind_ack(cotenant_bind.header.corr),
16302                )
16303                .await
16304                .unwrap();
16305            assert!(cotenant_task.await.unwrap().is_empty());
16306            let (cotenant_channel, cotenant_epoch) =
16307                published_route(&cotenant_rx.recv().await.unwrap());
16308
16309            // A second route.open, relayed to the incumbent and not yet acked.
16310            let caller = ConnectionId::new(32);
16311            let (caller_ctx, mut caller_rx) = route_ctx(caller);
16312            let (caller_task, caller_bind) = relay_route_open(
16313                &handler,
16314                caller,
16315                &caller_ctx.egress,
16316                &mut swap.incumbent_rx,
16317                101,
16318                "aft",
16319                "swap-caller",
16320            )
16321            .await;
16322            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16323
16324            register_candidate(&swap, None);
16325            cutover(&swap);
16326
16327            // The incumbent acks after promotion and before any drain.
16328            let ack = handler
16329                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16330                .await;
16331            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16332            if module_loop_error.is_some() {
16333                // What the connection loop does with an untranslated router
16334                // error: end the connection, releasing every route on it.
16335                handler.cleanup_connection(INCUMBENT).unwrap();
16336            }
16337
16338            // 1. The incumbent's other routes survive.
16339            assert!(
16340                cotenant_rx.try_recv().is_err(),
16341                "the co-tenant route on the incumbent was torn down by one late ack: \
16342                 {module_loop_error:?}"
16343            );
16344            assert!(matches!(
16345                swap.forwarding
16346                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16347                    .unwrap(),
16348                DataRoute::Client(DataRouteState::Bound(_))
16349            ));
16350            assert_eq!(module_loop_error, None);
16351            assert!(swap
16352                .registry
16353                .get_module_by_connection(INCUMBENT)
16354                .unwrap()
16355                .is_some());
16356
16357            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
16358            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16359                .await
16360                .expect("the incumbent is told to drop the abandoned binding")
16361                .unwrap()
16362                .frame;
16363            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16364            assert_eq!(goodbye.header.channel, abandoned_channel);
16365            assert_eq!(goodbye.header.epoch, abandoned_epoch);
16366            assert!(swap.incumbent_rx.try_recv().is_err());
16367
16368            // 3. The waiting client gets a retryable refusal and no route.
16369            let response = caller_task.await.unwrap();
16370            assert_eq!(response.len(), 1);
16371            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16372            assert!(caller_rx.try_recv().is_err());
16373
16374            // 4. The reservation pair is given back, and the pending bind
16375            //    settled exactly once: one accepted open (the co-tenant) and one
16376            //    refused open (the caller), nothing counted twice.
16377            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16378            let counters = handler.counters().snapshot();
16379            assert_eq!(
16380                keyed_total(&counters, "route_open_accepted_by_principal"),
16381                1
16382            );
16383            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16384            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16385        }
16386
16387        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
16388        /// id would resolve to the promoted candidate and every new route.open
16389        /// would be refused as reloading, leaving neither process routable.
16390        #[tokio::test]
16391        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16392            let mut swap = swap_with_incumbent().await;
16393            register_candidate(&swap, None);
16394            let incumbent = cutover(&swap);
16395            swap.forwarding
16396                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16397                .unwrap()
16398                .expect("the incumbent is still registered");
16399
16400            let client = ConnectionId::new(33);
16401            let (client_ctx, mut client_rx) = route_ctx(client);
16402            let route_handler = swap.handler.clone();
16403            let open_ctx = RouteCtx {
16404                connection_id: client,
16405                egress: client_ctx.egress.clone(),
16406            };
16407            let mut route_task = tokio::spawn(async move {
16408                route_handler
16409                    .handle_control_frame(
16410                        &open_ctx,
16411                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16412                    )
16413                    .await
16414                    .unwrap()
16415            });
16416            let bind = tokio::select! {
16417                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16418                response = &mut route_task => {
16419                    let response = response.unwrap();
16420                    panic!(
16421                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
16422                        parse_error(&response[0])["code"]
16423                    );
16424                }
16425            };
16426            swap.handler
16427                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16428                .await
16429                .unwrap();
16430            assert!(route_task.await.unwrap().is_empty());
16431            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16432            match swap
16433                .forwarding
16434                .lookup_data_route(client, channel, epoch)
16435                .unwrap()
16436            {
16437                DataRoute::Client(DataRouteState::Bound(route)) => {
16438                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16439                }
16440                other => panic!("expected a bound route on the candidate, got {other:?}"),
16441            }
16442            assert!(swap.incumbent_rx.try_recv().is_err());
16443        }
16444
16445        /// A candidate declares itself ready with `catalog.update` on its own
16446        /// connection. If the connection-keyed registry lookups searched only the
16447        /// active slot, this would answer `not_registered` and the candidate
16448        /// would never become ready.
16449        #[tokio::test]
16450        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16451            let swap = swap_with_incumbent().await;
16452            register_candidate(&swap, Some(false));
16453            let update = Frame::build(
16454                FrameType::Request,
16455                control_flags(),
16456                0,
16457                0,
16458                55,
16459                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16460                    provides: manifest("aft", PROTOCOL_VERSION).provides,
16461                    capabilities: None,
16462                    ready: Some(true),
16463                })
16464                .unwrap(),
16465            )
16466            .unwrap();
16467
16468            let replies = swap
16469                .handler
16470                .handle_control_frame(&swap.candidate_ctx, update)
16471                .await
16472                .unwrap();
16473
16474            assert_eq!(replies.len(), 1);
16475            assert_eq!(
16476                replies[0].header.ty,
16477                FrameType::Response,
16478                "candidate catalog.update was refused: {:?}",
16479                serde_json::from_slice::<Value>(&replies[0].body).ok()
16480            );
16481            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16482            assert_eq!(
16483                swap.registry
16484                    .get_module("aft")
16485                    .unwrap()
16486                    .unwrap()
16487                    .connection_id,
16488                INCUMBENT
16489            );
16490        }
16491    }
16492
16493    /// The HELLO gate while the supervisor has a swap open: only the nonce it
16494    /// minted for the candidate admits a second process, into the candidate
16495    /// slot, and that check runs ahead of the reserved-module gate.
16496    mod swap_admission {
16497        use super::*;
16498
16499        const INCUMBENT_NONCE: &str = "incumbent-nonce";
16500        const CANDIDATE_NONCE: &str = "candidate-nonce";
16501
16502        fn handler_with_incumbent(
16503            module_id: &str,
16504            reserved: bool,
16505        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16506            let registry = Arc::new(Registry::default());
16507            let supervisor = SupervisorHandle::new();
16508            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16509            if reserved {
16510                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16511            }
16512            let handler =
16513                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
16514            let incumbent = handler
16515                .handle_control(
16516                    ConnectionId::new(1),
16517                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16518                )
16519                .unwrap();
16520            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16521            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16522            (registry, supervisor, handler)
16523        }
16524
16525        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
16526        /// admits every nonce, so while a swap is open the swap gate is the only
16527        /// thing between a key-holder and the candidate slot. A nonce the
16528        /// supervisor did not mint, or none at all, is refused, and neither the
16529        /// incumbent's registration nor the candidate slot moves.
16530        #[test]
16531        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16532            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16533
16534            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16535                let replies = handler
16536                    .handle_control(
16537                        ConnectionId::new(connection),
16538                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16539                    )
16540                    .unwrap();
16541                assert_eq!(replies[0].header.ty, FrameType::Error);
16542                assert_eq!(
16543                    parse_error(&replies[0])["code"],
16544                    "swap_token_invalid",
16545                    "nonce {nonce:?}"
16546                );
16547            }
16548            assert!(registry.get_candidate("aft").unwrap().is_none());
16549            assert_eq!(
16550                registry.get_module("aft").unwrap().unwrap().connection_id,
16551                ConnectionId::new(1)
16552            );
16553
16554            // Control: the minted token is admitted, into the candidate slot,
16555            // and only once.
16556            let admitted = handler
16557                .handle_control(
16558                    ConnectionId::new(4),
16559                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16560                )
16561                .unwrap();
16562            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16563            assert_eq!(
16564                registry
16565                    .get_candidate("aft")
16566                    .unwrap()
16567                    .unwrap()
16568                    .connection_id,
16569                ConnectionId::new(4)
16570            );
16571            assert_eq!(
16572                registry.get_module("aft").unwrap().unwrap().connection_id,
16573                ConnectionId::new(1),
16574                "the candidate must not take the active slot"
16575            );
16576            let replayed = handler
16577                .handle_control(
16578                    ConnectionId::new(5),
16579                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16580                )
16581                .unwrap();
16582            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16583
16584            // The case only this gate covers: the incumbent has died mid-swap,
16585            // so its duplicate refusal is gone too, and without the gate a
16586            // key-holder would take the id's ACTIVE slot.
16587            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16588            let squatter = handler
16589                .handle_control(
16590                    ConnectionId::new(6),
16591                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16592                )
16593                .unwrap();
16594            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16595            assert!(
16596                registry.get_module("aft").unwrap().is_none(),
16597                "a squatter took the active slot of an id being swapped"
16598            );
16599        }
16600
16601        /// Design mutation arm (iii). A reserved module's candidate presents a
16602        /// nonce the reserved gate has never seen (that gate holds the
16603        /// incumbent's), so the swap gate must run first or the candidate is
16604        /// refused `reserved_module` and a reserved module can never be swapped.
16605        #[test]
16606        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16607            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16608
16609            let replies = handler
16610                .handle_control(
16611                    ConnectionId::new(2),
16612                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16613                )
16614                .unwrap();
16615
16616            assert_eq!(
16617                replies[0].header.ty,
16618                FrameType::HelloAck,
16619                "reserved candidate refused: {:?}",
16620                serde_json::from_slice::<Value>(&replies[0].body).ok()
16621            );
16622            assert_eq!(
16623                registry
16624                    .get_candidate("vault")
16625                    .unwrap()
16626                    .unwrap()
16627                    .connection_id,
16628                ConnectionId::new(2)
16629            );
16630        }
16631
16632        /// With no swap open the gate is inert: the incumbent's reserved gate
16633        /// and duplicate refusal behave exactly as before.
16634        #[test]
16635        fn without_an_open_swap_the_ordinary_gates_decide() {
16636            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16637            supervisor.close_swap("vault");
16638
16639            let candidate = handler
16640                .handle_control(
16641                    ConnectionId::new(2),
16642                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16643                )
16644                .unwrap();
16645            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16646            let duplicate = handler
16647                .handle_control(
16648                    ConnectionId::new(3),
16649                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16650                )
16651                .unwrap();
16652            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16653            assert!(registry.get_candidate("vault").unwrap().is_none());
16654        }
16655    }
16656
16657    /// `scope.sync` and `scope.describe` through the real control handler: who
16658    /// may sync is decided by the registration and launch nonce of the module
16659    /// connection, never by the request body.
16660    mod scopes {
16661        mod idle_tests {
16662            use super::*;
16663            include!("scope_idle_tests.rs");
16664        }
16665        mod route_open_timings {
16666            use super::*;
16667            include!("route_open_timing_tests.rs");
16668        }
16669        mod route_open_refusal_log {
16670            use super::*;
16671            include!("route_open_refusal_log_tests.rs");
16672        }
16673        mod activity {
16674            include!("activity_scope_tests.rs");
16675        }
16676        mod delta {
16677            include!("scope_handler_delta_tests.rs");
16678        }
16679        mod call_modes {
16680            include!("scope_call_mode_tests.rs");
16681        }
16682        #[tokio::test]
16683        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16684            let rig = rig().await;
16685            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16686                "s".into(),
16687                1,
16688                "tool".into(),
16689            ))
16690            .unwrap();
16691            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16692            let (direct, _rx) = wide_ctx(999);
16693            for (ctx, expected) in [
16694                (&rig.owner, FrameType::Response),
16695                (&direct, FrameType::Error),
16696            ] {
16697                let frame =
16698                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16699                        .unwrap();
16700                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16701                assert_eq!(replies[0].header.ty, expected);
16702                if expected == FrameType::Response {
16703                    let reply: subc_control::ScopeServedByReply =
16704                        serde_json::from_slice(&replies[0].body).unwrap();
16705                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16706                } else {
16707                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16708                }
16709            }
16710            let reply = call(
16711                &rig.handler,
16712                &rig.owner,
16713                &ModuleControlRequestFromModule::ScopeDescribe {
16714                    owner: Principal::Reserved {
16715                        module_id: OWNER.into(),
16716                    },
16717                    scope_ref: "s".into(),
16718                },
16719            )
16720            .await;
16721            assert!(matches!(
16722                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16723                ModuleControlResponseToModule::ScopeDescribe { .. }
16724            ));
16725        }
16726
16727        use subc_protocol::scope::{
16728            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16729            ScopeStatus,
16730        };
16731
16732        use super::*;
16733
16734        const OWNER: &str = "prefrontal-core";
16735
16736        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16737            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16738        }
16739
16740        async fn call(
16741            handler: &ControlHandler,
16742            ctx: &RouteCtx,
16743            request: &ModuleControlRequestFromModule,
16744        ) -> Frame {
16745            let body = serde_json::to_vec(request).unwrap();
16746            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16747            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16748            assert_eq!(replies.len(), 1, "{replies:?}");
16749            replies.pop().unwrap()
16750        }
16751
16752        async fn sync(
16753            handler: &ControlHandler,
16754            ctx: &RouteCtx,
16755            generation: u64,
16756            scopes: Vec<ScopeRecord>,
16757        ) -> Result<ModuleControlResponseToModule, String> {
16758            let reply = call(
16759                handler,
16760                ctx,
16761                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16762            )
16763            .await;
16764            match reply.header.ty {
16765                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16766                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16767            }
16768        }
16769
16770        async fn describe(
16771            handler: &ControlHandler,
16772            ctx: &RouteCtx,
16773            owner: &str,
16774            scope_ref: &str,
16775        ) -> ModuleControlResponseToModule {
16776            let reply = call(
16777                handler,
16778                ctx,
16779                &ModuleControlRequestFromModule::ScopeDescribe {
16780                    owner: Principal::Reserved {
16781                        module_id: owner.to_string(),
16782                    },
16783                    scope_ref: scope_ref.to_string(),
16784                },
16785            )
16786            .await;
16787            assert_eq!(
16788                reply.header.ty,
16789                FrameType::Response,
16790                "{:?}",
16791                parse_error(&reply)
16792            );
16793            serde_json::from_slice(&reply.body).unwrap()
16794        }
16795
16796        #[tokio::test]
16797        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16798            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16799            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16800            let mut rig = rig().await;
16801            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16802                .await;
16803            let _bound = rig
16804                .bound(
16805                    Some(AFT),
16806                    PLEXUS,
16807                    Some(
16808                        rig.selector("s", Some(1))
16809                            .with_bind_proof(Some(PROOF.into())),
16810                    ),
16811                )
16812                .await;
16813            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16814            let output = serde_json::to_string(&reply).unwrap();
16815            assert!(output.contains("\"status\":\"live\""));
16816            assert!(!output.contains(PROOF));
16817            assert!(!output.contains(HASH));
16818        }
16819
16820        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16821        async fn module(
16822            handler: &ControlHandler,
16823            connection: u64,
16824            module_id: &str,
16825            nonce: Option<&str>,
16826        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16827            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16828            hello_via_sink(
16829                handler,
16830                &ctx,
16831                &mut rx,
16832                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16833            )
16834            .await;
16835            (ctx, rx)
16836        }
16837
16838        /// `direct` and every other client connection has no registration, so
16839        /// it can neither sync nor own a scope.
16840        #[tokio::test]
16841        async fn a_client_connection_cannot_sync_or_describe() {
16842            let handler = ControlHandler::new(Arc::new(Registry::default()));
16843            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16844            for request in [
16845                ModuleControlRequestFromModule::ScopeSync {
16846                    generation: 1,
16847                    scopes: vec![head("s", 1)],
16848                },
16849                ModuleControlRequestFromModule::ScopeDescribe {
16850                    owner: Principal::Direct,
16851                    scope_ref: "s".to_string(),
16852                },
16853            ] {
16854                let reply = call(&handler, &ctx, &request).await;
16855                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16856            }
16857            assert!(
16858                !handler
16859                    .scopes
16860                    .read()
16861                    .unwrap()
16862                    .describe(
16863                        &Principal::Reserved {
16864                            module_id: OWNER.to_string()
16865                        },
16866                        "s"
16867                    )
16868                    .owner_synced
16869            );
16870        }
16871
16872        /// A module the supervisor did not spawn registers without a launch
16873        /// nonce, so it is never an owner's current launch.
16874        #[tokio::test]
16875        async fn a_module_without_a_supervised_launch_cannot_sync() {
16876            let handler = ControlHandler::new(Arc::new(Registry::default()));
16877            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16878            assert_eq!(
16879                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16880                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16881            );
16882        }
16883
16884        #[tokio::test]
16885        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16886            let supervisor = SupervisorHandle::new();
16887            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16888            let handler = ControlHandler::new(Arc::new(Registry::default()))
16889                .with_supervisor(supervisor.clone());
16890            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16891            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16892                .await
16893                .expect("the current launch syncs");
16894
16895            // A swap candidate registers with the swap token and is refused
16896            // while the incumbent keeps syncing.
16897            supervisor.open_swap(OWNER, "n2".to_string());
16898            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16899            assert_eq!(
16900                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16901                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16902            );
16903            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16904                .await
16905                .expect("the serving owner syncs during the swap");
16906
16907            // The swap fails and is rolled back. The candidate never held sync
16908            // authority, and still cannot sync.
16909            supervisor.close_swap(OWNER);
16910            assert_eq!(
16911                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16912                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16913            );
16914            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16915                .await
16916                .expect("the serving owner syncs after the rollback");
16917            handler.cleanup_connection(candidate.connection_id).unwrap();
16918
16919            // A swap that cuts over. Promotion records the candidate's nonce as
16920            // the module's spawn nonce, which is what `set_spawn_nonce` does
16921            // here; the promoted connection then takes authority at any
16922            // generation and the superseded incumbent is refused.
16923            supervisor.open_swap(OWNER, "n3".to_string());
16924            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16925            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16926            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16927                .await
16928                .expect("the promoted launch takes authority");
16929            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16930                panic!("unexpected reply {reply:?}");
16931            };
16932            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16933            assert_eq!(
16934                sync(&handler, &incumbent, 4, Vec::new()).await,
16935                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16936            );
16937        }
16938
16939        /// Authority dies with its connection: the cleanup path releases it,
16940        /// so the owner's next connection takes it at any generation.
16941        #[tokio::test]
16942        async fn closing_the_authority_connection_frees_sync_authority() {
16943            let supervisor = SupervisorHandle::new();
16944            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16945            let handler = ControlHandler::new(Arc::new(Registry::default()))
16946                .with_supervisor(supervisor.clone());
16947            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16948            sync(&handler, &first, 10, vec![head("s", 1)])
16949                .await
16950                .unwrap();
16951            handler.cleanup_connection(first.connection_id).unwrap();
16952
16953            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16954            sync(&handler, &second, 1, vec![head("s", 1)])
16955                .await
16956                .expect("the next connection takes the released authority");
16957        }
16958
16959        #[tokio::test]
16960        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16961            let supervisor = SupervisorHandle::new();
16962            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16963            let handler =
16964                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16965            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16966            sync(&handler, &first, 10, vec![head("s", 1)])
16967                .await
16968                .unwrap();
16969            handler
16970                .handle_control_frame(
16971                    &first,
16972                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16973                )
16974                .await
16975                .unwrap();
16976            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16977            sync(&handler, &second, 1, vec![head("s", 1)])
16978                .await
16979                .expect("GOODBYE releases authority even if the old socket remains open");
16980        }
16981
16982        #[tokio::test]
16983        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16984            let registry = Arc::new(Registry::default());
16985            let supervisor_handle = SupervisorHandle::new();
16986            let supervisor =
16987                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16988                    .with_handle(supervisor_handle.clone())
16989                    .with_daemon_incarnation("incarnation-7".to_string());
16990            // Configured with enabled: false, so the supervisor lists the
16991            // module without spawning a process for it.
16992            supervisor
16993                .supervise_configured(
16994                    ModuleSpec {
16995                        module_id: OWNER.to_string(),
16996                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16997                        args: Vec::new(),
16998                        env: Vec::new(),
16999                        reserved: false,
17000                        reserved_prefixes: Vec::new(),
17001                        protocol: ModuleProtocol::Subc,
17002                        overlap: Default::default(),
17003                    },
17004                    false,
17005                )
17006                .unwrap();
17007            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17008            let handler =
17009                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17010            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17011
17012            // Configured but not yet synced: a reader waits for the owner.
17013            let ModuleControlResponseToModule::ScopeDescribe {
17014                status,
17015                daemon_incarnation,
17016                owner_synced,
17017                owner_configured,
17018                scope,
17019                ..
17020            } = describe(&handler, &reader, OWNER, "s").await
17021            else {
17022                panic!("not a describe reply");
17023            };
17024            assert_eq!(status, ScopeStatus::NotLive);
17025            assert_eq!(daemon_incarnation, "incarnation-7");
17026            assert!(!owner_synced);
17027            assert!(owner_configured);
17028            assert!(scope.is_none());
17029
17030            // Not a supervised module: the owner will never sync, and a reader
17031            // refuses rather than waits.
17032            let ModuleControlResponseToModule::ScopeDescribe {
17033                status,
17034                owner_configured,
17035                ..
17036            } = describe(&handler, &reader, "ghost", "s").await
17037            else {
17038                panic!("not a describe reply");
17039            };
17040            assert_eq!(status, ScopeStatus::NotLive);
17041            assert!(!owner_configured);
17042
17043            // Live, with the stamp fields and the computed owner_authorized.
17044            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17045            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17046            let ModuleControlResponseToModule::ScopeDescribe {
17047                status,
17048                scope_epoch,
17049                owner_synced,
17050                scope,
17051                ..
17052            } = describe(&handler, &reader, OWNER, "s").await
17053            else {
17054                panic!("not a describe reply");
17055            };
17056            assert_eq!(status, ScopeStatus::Live);
17057            assert_eq!(scope_epoch, Some(4));
17058            assert!(owner_synced);
17059            let stamp = scope.expect("a live scope carries its stamp");
17060            assert!(
17061                stamp.owner_authorized,
17062                "prefrontal-core is the default authority"
17063            );
17064            assert_eq!(stamp.kind, ScopeKind::Head);
17065        }
17066
17067        #[tokio::test]
17068        async fn scope_authority_owners_decides_owner_authorized() {
17069            let supervisor = SupervisorHandle::new();
17070            supervisor.set_spawn_nonce("broca", "b1".to_string());
17071            let handler = ControlHandler::new(Arc::new(Registry::default()))
17072                .with_supervisor(supervisor)
17073                .with_scope_authority_owners(vec!["broca".to_string()]);
17074            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17075            let mut gated = head("s", 1);
17076            gated.attributes.agent_id = Some("agent".to_string());
17077            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17078            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17079                describe(&handler, &broca, "broca", "s").await
17080            else {
17081                panic!("not a describe reply");
17082            };
17083            assert!(scope.unwrap().owner_authorized);
17084        }
17085
17086        /// With route admission, the stamp, the commit re-check and drains in
17087        /// place, the feature is advertised: the module ops in HELLO_ACK, and
17088        /// `scopes/v1` in HELLO_ACK and `server.describe`.
17089        #[tokio::test]
17090        async fn scope_ops_and_the_scopes_capability_are_advertised() {
17091            let handler = ControlHandler::new(Arc::new(Registry::default()));
17092            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17093            let ack = hello_via_sink(
17094                &handler,
17095                &ctx,
17096                &mut rx,
17097                hello_frame("m", PROTOCOL_VERSION, 1),
17098            )
17099            .await;
17100            let ack = parse_ack(&ack);
17101            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17102                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17103            }
17104            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17105
17106            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17107            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17108            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17109            let reply = handler
17110                .handle_control_frame(&client, frame)
17111                .await
17112                .unwrap()
17113                .pop()
17114                .unwrap();
17115            let ClientControlResponse::ServerDescribe { capabilities, .. } =
17116                serde_json::from_slice(&reply.body).unwrap()
17117            else {
17118                panic!("not a server.describe reply");
17119            };
17120            assert!(
17121                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17122                "{capabilities:?}"
17123            );
17124        }
17125
17126        // ---- route admission, stamps, commit re-check and drains ----------
17127
17128        const PLEXUS: &str = "plexus";
17129        const OTHER: &str = "other";
17130        const AFT: &str = "aft";
17131        const BROCA: &str = "broca";
17132        const MAGIC: &str = "magic-context";
17133
17134        fn nonce(module_id: &str) -> String {
17135            format!("nonce-{module_id}")
17136        }
17137
17138        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17139            let (tx, rx) = mpsc::channel(64);
17140            (
17141                RouteCtx {
17142                    connection_id: ConnectionId::new(connection),
17143                    egress: FrameSink::new(tx),
17144                },
17145                rx,
17146            )
17147        }
17148
17149        /// A daemon with a configured owner (prefrontal-core) registered on its
17150        /// own module connection, two routable targets (plexus, other), and
17151        /// launch nonces minted for the modules that open routes as carriers.
17152        struct Rig {
17153            handler: ControlHandler,
17154            forwarding: Arc<ForwardingTable>,
17155            owner: RouteCtx,
17156            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17157            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17158            generation: u64,
17159            next_connection: u64,
17160            _supervisor: Supervisor,
17161        }
17162
17163        async fn rig() -> Rig {
17164            rig_with_flow_support(true).await
17165        }
17166
17167        async fn rig_with_flow_support(flow_support: bool) -> Rig {
17168            rig_with_scope_capabilities(flow_support, false).await
17169        }
17170
17171        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17172            let mut provides = Vec::new();
17173            if flow_support {
17174                provides.push("flow-scopes/v1");
17175            }
17176            if run_support {
17177                provides.push("agent-run-scopes/v1");
17178            }
17179            rig_with_target_capabilities(&provides).await
17180        }
17181
17182        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17183            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17184            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17185            // Decoder versions alone cannot admit scoped calls: only a capability
17186            // declaration promises the required run or flow behaviour.
17187            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17188            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17189            Frame::build(
17190                FrameType::Hello,
17191                control_flags(),
17192                0,
17193                0,
17194                connection,
17195                serde_json::to_vec(&body).unwrap(),
17196            )
17197            .unwrap()
17198        }
17199
17200        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17201            let registry = Arc::new(Registry::default());
17202            let forwarding = Arc::new(ForwardingTable::default());
17203            let supervisor_handle = SupervisorHandle::new();
17204            let supervisor =
17205                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17206                    .with_handle(supervisor_handle.clone());
17207            supervisor
17208                .supervise_configured(
17209                    ModuleSpec {
17210                        module_id: OWNER.to_string(),
17211                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17212                        args: Vec::new(),
17213                        env: Vec::new(),
17214                        reserved: false,
17215                        reserved_prefixes: Vec::new(),
17216                        protocol: ModuleProtocol::Subc,
17217                        overlap: Default::default(),
17218                    },
17219                    false,
17220                )
17221                .unwrap();
17222            for module_id in [OWNER, AFT, BROCA, MAGIC] {
17223                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17224            }
17225            let handler =
17226                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17227                    .with_supervisor(supervisor_handle);
17228            let (owner, mut owner_rx) = wide_ctx(1);
17229            hello_via_sink(
17230                &handler,
17231                &owner,
17232                &mut owner_rx,
17233                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17234            )
17235            .await;
17236            let mut modules = BTreeMap::new();
17237            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17238                let (ctx, mut rx) = wide_ctx(connection);
17239                let hello = hello_with_capabilities(module_id, connection, provides);
17240                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17241                modules.insert(module_id.to_string(), (ctx, rx));
17242            }
17243            Rig {
17244                handler,
17245                forwarding,
17246                owner,
17247                _owner_rx: owner_rx,
17248                modules,
17249                generation: 0,
17250                next_connection: 100,
17251                _supervisor: supervisor,
17252            }
17253        }
17254
17255        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17256            ScopeCarrier::new(Principal::Reserved {
17257                module_id: module_id.to_string(),
17258            })
17259            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17260        }
17261
17262        /// The scope most tests open under: aft carries to any module, broca
17263        /// only to plexus and other, and the owner delegates as agent-1.
17264        fn session(scope_epoch: u64) -> ScopeRecord {
17265            let mut record = head("s", scope_epoch);
17266            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17267            record.attributes.agent_id = Some("agent-1".to_string());
17268            record.attributes.delegates = true;
17269            record
17270        }
17271
17272        impl Rig {
17273            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17274                self.generation += 1;
17275                sync(&self.handler, &self.owner, self.generation, scopes)
17276                    .await
17277                    .expect("the owner's sync is accepted");
17278            }
17279
17280            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17281                ScopeSelector::new(
17282                    Principal::Reserved {
17283                        module_id: OWNER.to_string(),
17284                    },
17285                    scope_ref.to_string(),
17286                    scope_epoch,
17287                )
17288            }
17289
17290            fn open_frame(
17291                &mut self,
17292                opener: Option<&str>,
17293                target: &str,
17294                scope: Option<ScopeSelector>,
17295            ) -> (
17296                RouteCtx,
17297                mpsc::Receiver<crate::router::OutboundFrame>,
17298                Frame,
17299            ) {
17300                self.next_connection += 1;
17301                let (ctx, rx) = wide_ctx(self.next_connection);
17302                let root = unique_project_root("scoped-open");
17303                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17304                    target: RouteTarget::ToolProvider {
17305                        module_id: target.to_string(),
17306                    },
17307                    identity: BindIdentity::new(
17308                        root.path().to_path_buf(),
17309                        "unit".to_string(),
17310                        "session".to_string(),
17311                    ),
17312                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
17313                        module_id: module_id.to_string(),
17314                        launch_nonce: nonce(module_id),
17315                    }),
17316                    consumer_capabilities: None,
17317                    role_versions: None,
17318                    admission_facts: None,
17319                    scope,
17320                })
17321                .unwrap();
17322                let frame = Frame::build(
17323                    FrameType::Request,
17324                    control_flags(),
17325                    0,
17326                    0,
17327                    self.next_connection,
17328                    body,
17329                )
17330                .unwrap();
17331                (ctx, rx, frame)
17332            }
17333
17334            /// Open and expect a refusal before anything is relayed.
17335            async fn refused(
17336                &mut self,
17337                opener: Option<&str>,
17338                target: &str,
17339                scope: Option<ScopeSelector>,
17340            ) -> String {
17341                self.refusal_body(opener, target, scope).await["code"]
17342                    .as_str()
17343                    .unwrap()
17344                    .to_string()
17345            }
17346
17347            async fn refusal_body(
17348                &mut self,
17349                opener: Option<&str>,
17350                target: &str,
17351                scope: Option<ScopeSelector>,
17352            ) -> Value {
17353                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17354                let replies = tokio::time::timeout(
17355                    Duration::from_secs(2),
17356                    self.handler.handle_control_frame(&ctx, frame),
17357                )
17358                .await
17359                .expect("the open must be refused before waiting for a bind ack")
17360                .unwrap();
17361                assert_eq!(replies.len(), 1, "{replies:?}");
17362                assert_eq!(replies[0].header.ty, FrameType::Error);
17363                let (_, module_rx) = self.modules.get_mut(target).unwrap();
17364                assert!(
17365                    module_rx.try_recv().is_err(),
17366                    "a refused open relays nothing"
17367                );
17368                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17369                parse_error(&replies[0])
17370            }
17371
17372            /// Start an open and return its task and the bind the target got.
17373            async fn relayed(
17374                &mut self,
17375                opener: Option<&str>,
17376                target: &str,
17377                scope: Option<ScopeSelector>,
17378            ) -> Relayed {
17379                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17380                let handler = self.handler.clone();
17381                let task_ctx = ctx.clone();
17382                let task = tokio::spawn(async move {
17383                    handler
17384                        .handle_control_frame(&task_ctx, frame)
17385                        .await
17386                        .unwrap()
17387                });
17388                let (_, module_rx) = self.modules.get_mut(target).unwrap();
17389                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17390                    .await
17391                    .expect("the target receives the relayed route.bind")
17392                    .unwrap()
17393                    .frame;
17394                Relayed {
17395                    target: target.to_string(),
17396                    client: ctx,
17397                    client_rx: rx,
17398                    task,
17399                    bind,
17400                }
17401            }
17402
17403            async fn ack(&self, relayed: &Relayed) {
17404                let (module, _) = &self.modules[&relayed.target];
17405                self.handler
17406                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17407                    .await
17408                    .unwrap();
17409            }
17410
17411            /// Open, ack and return the bound route.
17412            async fn bound(
17413                &mut self,
17414                opener: Option<&str>,
17415                target: &str,
17416                scope: Option<ScopeSelector>,
17417            ) -> Bound {
17418                let relayed = self.relayed(opener, target, scope).await;
17419                self.ack(&relayed).await;
17420                let Relayed {
17421                    target,
17422                    client,
17423                    mut client_rx,
17424                    task,
17425                    bind,
17426                } = relayed;
17427                assert!(
17428                    task.await.unwrap().is_empty(),
17429                    "the open is answered by commit"
17430                );
17431                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17432                Bound {
17433                    target,
17434                    client,
17435                    client_rx,
17436                    channel,
17437                    epoch,
17438                    bind,
17439                }
17440            }
17441
17442            fn live(&self, route: &Bound) -> bool {
17443                matches!(
17444                    self.forwarding
17445                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17446                        .unwrap(),
17447                    DataRoute::Client(DataRouteState::Bound(_))
17448                )
17449            }
17450        }
17451
17452        struct Relayed {
17453            target: String,
17454            client: RouteCtx,
17455            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17456            task: tokio::task::JoinHandle<Vec<Frame>>,
17457            bind: Frame,
17458        }
17459
17460        struct Bound {
17461            target: String,
17462            client: RouteCtx,
17463            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17464            channel: u16,
17465            epoch: u32,
17466            bind: Frame,
17467        }
17468
17469        impl Bound {
17470            /// The reason of the `route.closed` this client was sent, after
17471            /// checking it also got a GOODBYE on exactly this route.
17472            fn closed_reason(&mut self) -> RouteCloseReason {
17473                let mut reason = None;
17474                let mut goodbye = false;
17475                while let Ok(outbound) = self.client_rx.try_recv() {
17476                    let frame = outbound.frame;
17477                    match frame.header.ty {
17478                        FrameType::Goodbye => {
17479                            assert_eq!(
17480                                (frame.header.channel, frame.header.epoch),
17481                                (self.channel, self.epoch)
17482                            );
17483                            goodbye = true;
17484                        }
17485                        FrameType::Push => {
17486                            let ClientControlPush::RouteClosed {
17487                                reason: r,
17488                                module_id,
17489                                ..
17490                            } = serde_json::from_slice(&frame.body).unwrap()
17491                            else {
17492                                panic!("unexpected push");
17493                            };
17494                            assert_eq!(module_id, self.target);
17495                            reason = Some(r);
17496                        }
17497                        other => panic!("unexpected frame {other:?}"),
17498                    }
17499                }
17500                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17501                reason.expect("the client is told why the route closed")
17502            }
17503
17504            fn untouched(&mut self) -> bool {
17505                self.client_rx.try_recv().is_err()
17506            }
17507
17508            fn stamp(&self) -> Option<ScopeStamp> {
17509                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17510                    ModuleControlRequest::RouteBind { scope, .. } => scope,
17511                    other => panic!("expected a route.bind, got {other:?}"),
17512                }
17513            }
17514        }
17515
17516        #[tokio::test]
17517        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17518        ) {
17519            let mut rig = rig().await;
17520            let mut record = session(1);
17521            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17522            record.child_owners = vec![Principal::Reserved {
17523                module_id: MAGIC.to_string(),
17524            }];
17525            rig.sync(vec![record]).await;
17526            let scope = || Some(rig_selector("s", Some(1)));
17527
17528            // Admitted: the owner, a bare carrier to any module, a targeted
17529            // carrier to its listed module.
17530            rig.bound(Some(OWNER), PLEXUS, scope()).await;
17531            rig.bound(Some(AFT), OTHER, scope()).await;
17532            rig.bound(Some(BROCA), PLEXUS, scope()).await;
17533
17534            // Refused scope_not_carrier: a targeted carrier to an unlisted
17535            // module, a module that is not listed at all (a child owner is not
17536            // a carrier), and a direct key-holder.
17537            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17538                assert_eq!(
17539                    rig.refused(opener, target, scope()).await,
17540                    error_codes::SCOPE_NOT_CARRIER,
17541                    "{opener:?} -> {target}"
17542                );
17543            }
17544        }
17545
17546        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17547            ScopeSelector::new(
17548                Principal::Reserved {
17549                    module_id: OWNER.to_string(),
17550                },
17551                scope_ref.to_string(),
17552                scope_epoch,
17553            )
17554        }
17555
17556        #[tokio::test]
17557        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17558            let mut rig = rig().await;
17559            rig.sync(vec![session(1)]).await;
17560            for opener in [OWNER, AFT] {
17561                assert_eq!(
17562                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17563                        .await,
17564                    error_codes::SCOPE_EPOCH_REQUIRED,
17565                    "{opener}"
17566                );
17567            }
17568        }
17569
17570        #[tokio::test]
17571        async fn admission_separates_not_synced_not_live_and_ended() {
17572            let mut rig = rig().await;
17573            // Before the configured owner's first sync: retryable.
17574            let code = rig
17575                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17576                .await;
17577            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17578            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17579
17580            // An owner that is not configured will never sync: terminal.
17581            let ghost = ScopeSelector::new(
17582                Principal::Reserved {
17583                    module_id: "ghost".to_string(),
17584                },
17585                "s".to_string(),
17586                Some(1),
17587            );
17588            assert_eq!(
17589                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17590                error_codes::SCOPE_NOT_LIVE
17591            );
17592
17593            rig.sync(vec![session(2)]).await;
17594            assert_eq!(
17595                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17596                    .await,
17597                error_codes::SCOPE_NOT_LIVE
17598            );
17599            for epoch in [1, 3] {
17600                assert_eq!(
17601                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17602                        .await,
17603                    error_codes::SCOPE_ENDED,
17604                    "epoch {epoch}"
17605                );
17606            }
17607            // Control: the live epoch is admitted.
17608            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17609                .await;
17610        }
17611
17612        #[tokio::test]
17613        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17614            let mut rig = rig().await;
17615            rig.sync(vec![session(1)]).await;
17616            let route = rig
17617                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17618                .await;
17619            let stamp = route.stamp().expect("a scoped bind carries the stamp");
17620            assert_eq!(stamp.scope_ref, "s");
17621            assert_eq!(stamp.scope_epoch, 1);
17622            assert_eq!(stamp.kind, ScopeKind::Head);
17623            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17624            assert!(stamp.attributes.delegates);
17625            assert!(stamp.owner_authorized);
17626            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17627            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17628
17629            // broca owns a scope of its own on its own module connection; it is
17630            // not in scope_authority_owners, so its stamp is not authorized.
17631            let (broca, mut broca_rx) = wide_ctx(50);
17632            hello_via_sink(
17633                &rig.handler,
17634                &broca,
17635                &mut broca_rx,
17636                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17637            )
17638            .await;
17639            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17640                .await
17641                .unwrap();
17642            let own = ScopeSelector::new(
17643                Principal::Reserved {
17644                    module_id: BROCA.to_string(),
17645                },
17646                "b".to_string(),
17647                Some(1),
17648            );
17649            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17650            assert!(!route.stamp().unwrap().owner_authorized);
17651        }
17652
17653        #[tokio::test]
17654        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17655            let mut rig = rig().await;
17656            let mut record = head("s", 1);
17657            record.carriers = vec![carrier(AFT, None)];
17658            let flow_id = "Flow:run-7/step_2!~";
17659            record.attributes.flow_id = Some(flow_id.to_string());
17660            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17661                .await
17662                .unwrap();
17663            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17664                panic!("not a sync reply");
17665            };
17666            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17667            let route = rig
17668                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17669                .await;
17670            assert!(rig.live(&route), "the stamped bind committed");
17671            let stamp = route.stamp().expect("a flow scope carries a stamp");
17672            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17673            assert_eq!(stamp.attributes.agent_id, None);
17674            assert!(!stamp.attributes.delegates);
17675            assert!(stamp.owner_authorized);
17676            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17677            assert_eq!(unscoped.stamp(), None);
17678        }
17679
17680        #[tokio::test]
17681        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17682            let mut rig = rig_with_flow_support(false).await;
17683            let mut record = session(1);
17684            record.attributes.flow_id = Some("flow:7".to_string());
17685            rig.sync(vec![record]).await;
17686            for opener in [OWNER, AFT] {
17687                let body = rig
17688                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17689                    .await;
17690                assert_eq!(body["code"], "target_flow_unsupported");
17691                let message = body["message"].as_str().unwrap();
17692                for required in [PLEXUS, "flow-scopes/v1"] {
17693                    assert!(message.contains(required), "{message}");
17694                }
17695            }
17696        }
17697
17698        #[tokio::test]
17699        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17700            let mut rig = rig_with_flow_support(true).await;
17701            let mut record = session(1);
17702            record.attributes.flow_id = Some("flow:7".to_string());
17703            rig.sync(vec![record]).await;
17704            for opener in [OWNER, AFT] {
17705                let route = rig
17706                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17707                    .await;
17708                assert!(rig.live(&route));
17709                assert_eq!(
17710                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17711                    Some("flow:7")
17712                );
17713            }
17714        }
17715
17716        #[tokio::test]
17717        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17718            use std::future::Future;
17719
17720            let mut rig = rig_with_flow_support(true).await;
17721            let mut record = session(1);
17722            record.attributes.flow_id = Some("flow:7".to_string());
17723            rig.sync(vec![record]).await;
17724            let (client, mut client_rx, frame) =
17725                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17726            // Hold the route-open response permit so admission sees the first
17727            // target but relay reservation cannot capture an endpoint yet.
17728            for _ in 0..64 {
17729                client.egress.try_send(route_bind_ack(1)).unwrap();
17730            }
17731            let handler = rig.handler.clone();
17732            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17733            std::future::poll_fn(|cx| {
17734                assert!(open.as_mut().poll(cx).is_pending());
17735                std::task::Poll::Ready(())
17736            })
17737            .await;
17738            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17739
17740            let old_connection = rig.modules[PLEXUS].0.connection_id;
17741            rig.handler.cleanup_connection(old_connection).unwrap();
17742            let (replacement, mut replacement_rx) = wide_ctx(200);
17743            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17744            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17745            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17746            let hello = Frame::build(
17747                FrameType::Hello,
17748                control_flags(),
17749                0,
17750                0,
17751                200,
17752                serde_json::to_vec(&body).unwrap(),
17753            )
17754            .unwrap();
17755            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17756
17757            client_rx.try_recv().unwrap();
17758            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17759                .await
17760                .expect("the replacement is refused without waiting for a bind ack")
17761                .unwrap();
17762            assert_eq!(replies.len(), 1);
17763            let body = parse_error(&replies[0]);
17764            assert_eq!(body["code"], "target_flow_unsupported");
17765            for required in [PLEXUS, "flow-scopes/v1"] {
17766                assert!(body["message"].as_str().unwrap().contains(required));
17767            }
17768            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17769            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17770            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17771            assert!(rig
17772                .handler
17773                .registry
17774                .get_module_by_connection(replacement.connection_id)
17775                .unwrap()
17776                .is_some());
17777        }
17778
17779        #[tokio::test]
17780        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17781            use std::future::Future;
17782            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17783            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17784            const NEXT_HASH: &str =
17785                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17786            let mut rig = rig().await;
17787            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17788            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17789            rig.sync(vec![record.clone()]).await;
17790            let (client, mut rx, frame) = rig.open_frame(
17791                None,
17792                PLEXUS,
17793                Some(
17794                    rig.selector("s", Some(1))
17795                        .with_bind_proof(Some(PROOF.into())),
17796                ),
17797            );
17798            // Hold egress reservation while the scope owner changes its bind-proof
17799            // hash, so a proof valid at admission must be checked again at relay.
17800            for _ in 0..64 {
17801                client.egress.try_send(route_bind_ack(1)).unwrap();
17802            }
17803            let handler = rig.handler.clone();
17804            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17805            std::future::poll_fn(|cx| {
17806                assert!(open.as_mut().poll(cx).is_pending());
17807                std::task::Poll::Ready(())
17808            })
17809            .await;
17810            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17811            rig.sync(vec![record]).await;
17812            rx.try_recv().unwrap();
17813            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17814                .await
17815                .unwrap()
17816                .unwrap();
17817            assert_eq!(
17818                parse_error(&replies[0])["code"],
17819                error_codes::SCOPE_BIND_PROOF_MISMATCH
17820            );
17821            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17822            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17823            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17824        }
17825
17826        #[tokio::test]
17827        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17828            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17829            let mut rig = rig().await;
17830            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17831                .await;
17832            for opener in [OWNER, AFT] {
17833                for proof in [None, Some("malformed+proof=")] {
17834                    let route = rig
17835                        .bound(
17836                            Some(opener),
17837                            PLEXUS,
17838                            Some(
17839                                rig.selector("s", Some(1))
17840                                    .with_bind_proof(proof.map(str::to_string)),
17841                            ),
17842                        )
17843                        .await;
17844                    assert!(rig.live(&route));
17845                }
17846            }
17847        }
17848
17849        #[tokio::test]
17850        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17851            use std::future::Future;
17852            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17853            const NEXT_HASH: &str =
17854                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17855            let mut rig = rig().await;
17856            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17857                .await;
17858            let (client, mut rx, frame) =
17859                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17860            for _ in 0..64 {
17861                client.egress.try_send(route_bind_ack(1)).unwrap();
17862            }
17863            let handler = rig.handler.clone();
17864            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17865            std::future::poll_fn(|cx| {
17866                assert!(open.as_mut().poll(cx).is_pending());
17867                std::task::Poll::Ready(())
17868            })
17869            .await;
17870            rig.sync(vec![
17871                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17872            ])
17873            .await;
17874            rx.try_recv().unwrap();
17875            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17876            let bind = std::future::poll_fn(|cx| {
17877                assert!(open.as_mut().poll(cx).is_pending());
17878                module_rx.poll_recv(cx)
17879            })
17880            .await
17881            .unwrap()
17882            .frame;
17883            handler
17884                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17885                .await
17886                .unwrap();
17887            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17888                .await
17889                .unwrap()
17890                .unwrap();
17891            assert_eq!(replies.len(), 1);
17892            // Proof rechecking allowed the relay. Independent tag settlement
17893            // still asks the opener to retry after any same-epoch content change.
17894            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17895            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17896            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17897            let route = rig
17898                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17899                .await;
17900            assert!(rig.live(&route));
17901        }
17902
17903        #[tokio::test]
17904        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17905        {
17906            let mut rig = rig_with_flow_support(false).await;
17907            rig.sync(vec![session(1)]).await;
17908            let route = rig
17909                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17910                .await;
17911            assert!(rig.live(&route));
17912            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17913            let mut record = session(1);
17914            record.attributes.flow_id = Some("flow:7".to_string());
17915            rig.sync(vec![record]).await;
17916            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17917            assert!(rig.live(&unscoped));
17918            assert_eq!(unscoped.stamp(), None);
17919        }
17920
17921        #[tokio::test]
17922        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17923            let mut rig = rig().await;
17924            let mut record = session(1);
17925            record.attributes.flow_id = Some("flow:7".to_string());
17926            rig.sync(vec![record.clone()]).await;
17927            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17928            let mut owner_route = rig
17929                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17930                .await;
17931            let mut carrier_route = rig
17932                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17933                .await;
17934            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17935            rig.sync(vec![record.clone()]).await;
17936            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17937            assert!(rig.live(&owner_route) && owner_route.untouched());
17938            assert!(rig.live(&carrier_route) && carrier_route.untouched());
17939
17940            record.attributes.flow_id = Some("flow:8".to_string());
17941            rig.sync(vec![record]).await;
17942            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17943            let before = before.unwrap();
17944            assert_eq!(after.scope_epoch, before.scope_epoch);
17945            assert!(after.version > before.version);
17946            for route in [&mut owner_route, &mut carrier_route] {
17947                assert!(!rig.live(route));
17948                assert_eq!(
17949                    route.closed_reason(),
17950                    RouteCloseReason::ScopeDelegationChanged
17951                );
17952            }
17953            assert!(rig.live(&unscoped) && unscoped.untouched());
17954            // Each provider also receives a GOODBYE for its drained route;
17955            // consume it before expecting the next route.bind on that sink.
17956            for target in [PLEXUS, OTHER] {
17957                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17958                let goodbye = module_rx
17959                    .try_recv()
17960                    .expect("the provider sees the drain")
17961                    .frame;
17962                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17963                assert!(module_rx.try_recv().is_err());
17964            }
17965            let rebound = rig
17966                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17967                .await;
17968            assert_eq!(
17969                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17970                Some("flow:8")
17971            );
17972        }
17973
17974        /// The owner's sync lands between admission and the module's ack. The
17975        /// open is refused by name, the module's other routes stay up, and the
17976        /// reserved pair is released. Changed content is retryable; an ended
17977        /// scope is not.
17978        #[tokio::test]
17979        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17980            let mut rig = rig().await;
17981            rig.sync(vec![session(1)]).await;
17982            let mut cotenant = rig
17983                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17984                .await;
17985
17986            let mut changed = session(1);
17987            changed.child_owners.push(Principal::Reserved {
17988                module_id: MAGIC.to_string(),
17989            });
17990            let mut ended = None;
17991            for (code, next) in [
17992                (error_codes::SCOPE_CHANGED, vec![changed]),
17993                (error_codes::SCOPE_ENDED, Vec::new()),
17994            ] {
17995                let relayed = rig
17996                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17997                    .await;
17998                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17999                ended = Some(next.is_empty());
18000                rig.sync(next).await;
18001                rig.ack(&relayed).await;
18002                let replies = relayed.task.await.unwrap();
18003                assert_eq!(replies.len(), 1, "{replies:?}");
18004                assert_eq!(parse_error(&replies[0])["code"], code);
18005                assert_eq!(
18006                    subc_protocol::error_codes::is_retryable_route_open(code),
18007                    code == error_codes::SCOPE_CHANGED
18008                );
18009                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18010                // The module is told to drop just the binding it created.
18011                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18012                // Collected, because ending the scope also closes the co-tenant
18013                // route, whose GOODBYE comes first.
18014                let mut goodbyes = Vec::new();
18015                while let Ok(outbound) = plexus_rx.try_recv() {
18016                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18017                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18018                }
18019                assert!(
18020                    goodbyes.contains(&(bind_channel, bind_epoch)),
18021                    "{goodbyes:?}"
18022                );
18023                assert!(rig
18024                    .handler
18025                    .registry
18026                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18027                    .unwrap()
18028                    .is_some());
18029            }
18030            assert_eq!(ended, Some(true));
18031            // The co-tenant stayed up through the change, and closed only when
18032            // the scope ended, by the drain rule rather than by the commit.
18033            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18034        }
18035
18036        /// Each row of the drain table on one set of routes: the owner's, a
18037        /// bare carrier's, and a targeted carrier's to each of its targets.
18038        #[tokio::test]
18039        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18040            struct Case {
18041                name: &'static str,
18042                change: fn(&mut ScopeRecord),
18043                /// Closed routes by index: owner->plexus, aft->plexus,
18044                /// broca->plexus, broca->other.
18045                closed: [Option<RouteCloseReason>; 4],
18046            }
18047            use RouteCloseReason::*;
18048            let cases = [
18049                Case {
18050                    name: "a carrier entry removed",
18051                    change: |r| {
18052                        r.carriers.retain(|c| {
18053                            c.principal
18054                                != Principal::Reserved {
18055                                    module_id: AFT.to_string(),
18056                                }
18057                        })
18058                    },
18059                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18060                },
18061                Case {
18062                    name: "a target removed from a carrier",
18063                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18064                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
18065                },
18066                Case {
18067                    name: "a bare carrier narrowed to targets",
18068                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18069                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18070                },
18071                Case {
18072                    name: "delegates turned off",
18073                    change: |r| r.attributes.delegates = false,
18074                    closed: [Some(ScopeDelegationChanged); 4],
18075                },
18076                Case {
18077                    name: "agent_id changed",
18078                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18079                    closed: [Some(ScopeDelegationChanged); 4],
18080                },
18081                Case {
18082                    name: "a carrier added, child owners changed, the record re-sent",
18083                    change: |r| {
18084                        r.carriers.push(carrier(MAGIC, None));
18085                        r.child_owners.push(Principal::Reserved {
18086                            module_id: MAGIC.to_string(),
18087                        });
18088                    },
18089                    closed: [None; 4],
18090                },
18091                Case {
18092                    name: "a target added",
18093                    change: |r| {
18094                        r.carriers[1]
18095                            .targets
18096                            .as_mut()
18097                            .unwrap()
18098                            .push("third".to_string())
18099                    },
18100                    closed: [None; 4],
18101                },
18102                Case {
18103                    name: "delegates turned on",
18104                    change: |r| r.attributes.delegates = true,
18105                    closed: [None; 4],
18106                },
18107            ];
18108            for case in cases {
18109                let mut rig = rig().await;
18110                rig.sync(vec![session(1)]).await;
18111                let scope = || Some(rig_selector("s", Some(1)));
18112                let mut routes = [
18113                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
18114                    rig.bound(Some(AFT), PLEXUS, scope()).await,
18115                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
18116                    rig.bound(Some(BROCA), OTHER, scope()).await,
18117                ];
18118                let mut record = session(1);
18119                (case.change)(&mut record);
18120                rig.sync(vec![record]).await;
18121                for (index, expected) in case.closed.iter().enumerate() {
18122                    let route = &mut routes[index];
18123                    match expected {
18124                        Some(reason) => {
18125                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
18126                            assert_eq!(
18127                                route.closed_reason(),
18128                                *reason,
18129                                "{}: route {index}",
18130                                case.name
18131                            );
18132                        }
18133                        None => {
18134                            assert!(rig.live(route), "{}: route {index} closed", case.name);
18135                            assert!(
18136                                route.untouched(),
18137                                "{}: route {index} was told something",
18138                                case.name
18139                            );
18140                        }
18141                    }
18142                }
18143            }
18144        }
18145
18146        #[tokio::test]
18147        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18148            // Removed, and replaced by a higher epoch.
18149            for next in [Vec::new(), vec![session(2)]] {
18150                let mut rig = rig().await;
18151                rig.sync(vec![session(1)]).await;
18152                let mut route = rig
18153                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18154                    .await;
18155                rig.sync(next).await;
18156                assert!(!rig.live(&route));
18157                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18158            }
18159
18160            // A child whose parent ends: its routes close as parent-ended, the
18161            // child stays live, and routes under the parent close as ended.
18162            let mut rig = rig().await;
18163            let mut child = session(1);
18164            child.scope_ref = "child".to_string();
18165            child.kind = ScopeKind::Worker;
18166            child.parent = Some(ScopeParent::new(
18167                Principal::Reserved {
18168                    module_id: OWNER.to_string(),
18169                },
18170                "s".to_string(),
18171                1,
18172            ));
18173            rig.sync(vec![session(1), child.clone()]).await;
18174            let mut child_route = rig
18175                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18176                .await;
18177            assert_eq!(
18178                child_route.stamp().unwrap().parent_state,
18179                Some(ParentState::Linked)
18180            );
18181            rig.sync(vec![child]).await;
18182            assert!(!rig.live(&child_route));
18183            assert_eq!(
18184                child_route.closed_reason(),
18185                RouteCloseReason::ScopeParentEnded
18186            );
18187        }
18188
18189        #[tokio::test]
18190        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18191        ) {
18192            let mut rig = rig().await;
18193            rig.sync(vec![session(1)]).await;
18194            let mut route = rig
18195                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18196                .await;
18197            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18198            rig.sync(vec![session(1)]).await;
18199            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18200            assert!(rig.live(&route) && route.untouched());
18201
18202            // A call in flight on the route when another carrier is added. A
18203            // forwarded REQUEST holds one credit on the route's flow until the
18204            // module answers; the router takes it exactly like this.
18205            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18206                .forwarding
18207                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18208                .unwrap()
18209            else {
18210                panic!("the route is bound");
18211            };
18212            binding.flow.acquire_tagged(9, false).await.unwrap();
18213            let mut widened = session(1);
18214            widened.carriers.push(carrier(MAGIC, None));
18215            rig.sync(vec![widened]).await;
18216            assert!(rig.live(&route) && route.untouched());
18217            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18218            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18219            // The call's credit is still held on an open flow, so its answer
18220            // will be delivered: closing the route would have closed the flow.
18221            assert_eq!(binding.flow.in_flight(), 1);
18222            binding
18223                .flow
18224                .acquire_tagged(10, false)
18225                .await
18226                .expect("the flow is still open");
18227        }
18228
18229        /// A swap's superseded endpoint keeps its routes until drained; ending
18230        /// the scope closes them there too.
18231        #[tokio::test]
18232        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18233            let mut rig = rig().await;
18234            rig.sync(vec![session(1)]).await;
18235            let mut on_incumbent = rig
18236                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18237                .await;
18238
18239            // Swap plexus: register a candidate and cut over, leaving the
18240            // incumbent superseded with the route still on it.
18241            let (candidate, _candidate_rx) = wide_ctx(9);
18242            let registration = rig
18243                .handler
18244                .registry
18245                .register_candidate_with_control_ops(
18246                    manifest(PLEXUS, PROTOCOL_VERSION),
18247                    PROTOCOL_VERSION,
18248                    candidate.connection_id,
18249                    module_baseline_control_ops(),
18250                )
18251                .unwrap();
18252            rig.forwarding
18253                .register_candidate_module_connection(
18254                    candidate.connection_id,
18255                    PLEXUS.to_string(),
18256                    PROTOCOL_VERSION,
18257                    manifest_concurrency(&registration.manifest),
18258                    candidate.egress.clone(),
18259                )
18260                .unwrap();
18261            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18262            rig.handler
18263                .registry
18264                .promote_candidate(PLEXUS)
18265                .unwrap()
18266                .unwrap();
18267            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18268
18269            rig.sync(Vec::new()).await;
18270            assert!(!rig.live(&on_incumbent));
18271            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18272            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18273            let goodbye = incumbent_rx
18274                .try_recv()
18275                .expect("the superseded endpoint is told")
18276                .frame;
18277            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18278        }
18279    }
18280}
18281
18282#[cfg(test)]
18283mod concurrency_default_exposure_tests {
18284    use super::*;
18285
18286    fn hello_body(role_json: &str) -> Vec<u8> {
18287        format!(
18288            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18289        )
18290        .into_bytes()
18291    }
18292
18293    fn manifest_from(body: &[u8]) -> ModuleManifest {
18294        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18295        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18296            .expect("manifest parses")
18297    }
18298
18299    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18300
18301    #[test]
18302    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18303        let body = hello_body(&format!(
18304            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18305        ));
18306        let manifest = manifest_from(&body);
18307        // Precondition: serde really resolved it to the default, so the typed
18308        // manifest alone cannot answer the question this probe exists for.
18309        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18310        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18311    }
18312
18313    #[test]
18314    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18315        let body = hello_body(&format!(
18316            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18317        ));
18318        let manifest = manifest_from(&body);
18319        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18320        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18321    }
18322
18323    #[test]
18324    fn non_management_roles_are_never_reported() {
18325        let body = hello_body(
18326            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18327        );
18328        let manifest = manifest_from(&body);
18329        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18330    }
18331}