Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_confirm_operations,
25        validate_hello_event_declarations, validate_hello_flow_operations,
26        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29    },
30    peer::{
31        encode_route_bind, ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY,
32        OPERATOR_APP_CAPABILITY,
33    },
34    scope::{
35        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
36        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
37        SCOPE_SYNC_OP,
38    },
39    session::{
40        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
41        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
42        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
43        ROLE_VERSIONS_FIELD,
44    },
45    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
46    Priority, RouteTarget, PROTOCOL_VERSION,
47};
48use tokio::time::{timeout_at, Instant};
49use tracing::{debug, info, warn};
50
51use crate::{
52    caller_identity::CallerIdentity,
53    capability_requirements::{
54        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
55        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
56        RequirementStatus, RuntimeModule,
57    },
58    child_roster::DaemonShutdownFlag,
59    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
60    forwarding::{
61        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
62        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
63        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
64    },
65    observability::{
66        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
67    },
68    privileged_prompt::{prompt_text, PrivilegedOperation},
69    provenance::{
70        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
71    },
72    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
73    router::{daemon_error_flags, RouteCtx, RouterError},
74    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
75    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
76    stderr_tail::{CaptureState, TailEntry},
77    supervise::{
78        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
79        SupervisorHandle, SwapHelloAdmission,
80    },
81    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
82};
83
84/// Lowest envelope version this subc build will negotiate.
85///
86/// Module HELLO negotiation is exact: peers must use the daemon's locked
87/// protocol version. Older and newer peers receive `version_unsupported` and
88/// are not registered.
89pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
90
91const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
92const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
93const CAP_PING_PONG: &str = "ping_pong_v1";
94const CAP_SESSION_ATTACH: &str = "session_attach_v1";
95const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
96
97const SUBC_CONTROL_OPS: &[&str] = &[
98    ops::ACTIVITY_RECORD,
99    ops::ACTIVITY_QUERY,
100    ops::ACTIVITY_SUBSCRIBE,
101    ops::SERVER_DESCRIBE,
102    ops::CATALOG_LIST,
103    ops::ROUTE_OPEN,
104    ops::ROUTE_POLL,
105    ops::ROUTE_CLOSING,
106    ops::ROUTE_CLOSED,
107    ops::SUPERVISOR_LIST,
108    ops::SUPERVISOR_RESTART,
109    ops::SUPERVISOR_SWAP,
110    ops::SUPERVISOR_RELOAD,
111    ops::SUPERVISOR_RESCAN,
112    ops::SUPERVISOR_RELEASE_RESERVED,
113    ops::SUPERVISOR_SET_ENABLED,
114    ops::SUPERVISOR_HEALTH_PROBE,
115    ops::SUPERVISOR_HEALTH,
116    ops::SUPERVISOR_STDERR_TAIL,
117    ops::SUPERVISOR_TERMINALS,
118    ops::SUPERVISOR_ROUTES,
119    ops::SUPERVISOR_PROVENANCE,
120    ops::SUPERVISOR_SPAWN_SNAPSHOT,
121    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
122];
123
124/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
125/// announced by a capability in `server.describe` instead, and clients check
126/// that capability before sending it. `ck daemon --json` prints `subc_ops`
127/// verbatim and scripts read that output, so a new name there would change it.
128/// A malformed body for one of these is still refused `invalid_control_body`
129/// rather than as an unknown op.
130const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
131
132mod rescan_checked;
133use rescan_checked::{RescanMode, RescanPlan};
134
135const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
136    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
137    "supervisor.live_roots",
138    SCOPE_SYNC_OP,
139    SCOPE_APPLY_OP,
140    SCOPE_DESCRIBE_OP,
141    "operator.confirm",
142];
143
144/// Module-originated ops the daemon answers but does not advertise in
145/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
146/// belongs to is incomplete, so no module is told it works before it does.
147const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
148
149const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
150
151/// How long subc waits for a module to ack a relayed route.bind before returning
152/// `module_timeout`. The ack waits on the module's own configure, which for AFT
153/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
154/// and DB-open work — on a cold page cache or a large repo that legitimately
155/// exceeds a couple of seconds. The default is generous because rejecting a VALID
156/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
157/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
158pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
159
160/// How many CONSECUTIVE full-budget relay timeouts against one target module
161/// open that module's bind-relay breaker.
162///
163/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
164/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
165/// so three consecutive full-budget timeouts take ~36s to observe: every client
166/// whose open contributed to opening the breaker had already given up on its
167/// own. That is what makes opening the breaker unable to turn a call that would
168/// have succeeded into a refusal — it can only make an already-failing module
169/// fail faster.
170///
171/// Two would be reachable inside one default deadline. One would convict a
172/// module on a single cold-cache bind, which is exactly the valid-but-slow case
173/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
174pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
175
176/// How long a module's bind-relay breaker stays open before exactly one
177/// `route.open` is let through as a probe.
178///
179/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
180/// re-pays a full-budget stall almost continuously, and the breaker stops being
181/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
182/// deadline: a client that starts retrying after the module recovers has to get
183/// a probe opportunity inside its own deadline, or the breaker converts a
184/// recovered module into a failed call — the failure it exists to prevent,
185/// pointed the other way.
186///
187/// 20s sits between those with room on both sides, and it caps what a wedged
188/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
189/// rather than one per `route.open` per connection. The stall that motivated
190/// this, with its measurements, is written up in
191/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
192/// waited the whole budget out.
193pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
194
195const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
196const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
197
198fn reload_verdict(
199    configured: &Path,
200    spawned_from: Option<&Path>,
201    image: subc_control::RunningImageAgreement,
202) -> PendingReloadVerdict {
203    let path = match spawned_from {
204        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
205            ReloadPathAgreement::Match
206        }
207        Some(spawned_from) => ReloadPathAgreement::Mismatch {
208            configured: configured.to_path_buf(),
209            spawned_from: spawned_from.to_path_buf(),
210        },
211        None => ReloadPathAgreement::Unavailable {
212            reason: if matches!(
213                image,
214                subc_control::RunningImageAgreement::Unavailable {
215                    reason: subc_control::RunningImageUnavailableReason::NotRunning
216                }
217            ) {
218                ReloadPathUnavailableReason::NotRunning
219            } else {
220                ReloadPathUnavailableReason::SpawnedPathUnavailable
221            },
222        },
223    };
224    PendingReloadVerdict {
225        path,
226        image,
227        spec: None,
228    }
229}
230
231fn launch_spec_agreement(
232    configured: &crate::supervise::ModuleSpec,
233    running: bool,
234    spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237    if !running {
238        return Agreement::Unavailable {
239            reason: Reason::NotRunning,
240        };
241    }
242    match spawned {
243        None => Agreement::Unavailable {
244            reason: Reason::SpawnedSpecUnavailable,
245        },
246        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247            Agreement::Match
248        }
249        Some(_) => Agreement::Mismatch,
250    }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254    if configured == spawned_from {
255        return true;
256    }
257    #[cfg(windows)]
258    {
259        // Windows spawn records the resolved verbatim path (\\?\...), while
260        // configuration keeps its original spelling. Resolve both with the OS
261        // instead of stripping prefixes or folding case, which could conflate
262        // distinct names in case-sensitive directories. Different resolved
263        // paths still differ even if their files have identical content.
264        match (
265            std::fs::canonicalize(configured),
266            std::fs::canonicalize(spawned_from),
267        ) {
268            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269            _ => false,
270        }
271    }
272    #[cfg(not(windows))]
273    {
274        false
275    }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280    build: DaemonBuildProvenance,
281    pid: Option<u32>,
282    started_at_ms: Option<u64>,
283    start_clock: Option<crate::clock::StartClock>,
284    executable_path: Option<PathBuf>,
285    executable_identity: Option<SpawnedFileIdentity>,
286    process_start_time: Option<u64>,
287    probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291    fn default() -> Self {
292        Self {
293            build: DaemonBuildProvenance {
294                build_git_sha: None,
295                build_lock_digest: None,
296            },
297            pid: None,
298            started_at_ms: None,
299            start_clock: None,
300            executable_path: None,
301            executable_identity: None,
302            process_start_time: None,
303            probe: ExecutableIdentityProbe::default(),
304        }
305    }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310    supervisor: Supervisor,
311    config_path: PathBuf,
312    configured_port: Option<u16>,
313    storage_config: Option<crate::daemon_config::StorageConfig>,
314    admission_facts_carrier_module_id: Option<String>,
315    admission_facts_targets: Option<Vec<String>>,
316    scope_authority_owners: Vec<String>,
317    privileged_operator_prompt: PrivilegedOperatorPrompt,
318    /// Counts executed rescans that set out to change the supervised modules'
319    /// staged or applied configuration, counted before the first change so a
320    /// rescan that fails partway still counts. With the daemon incarnation it
321    /// names the state a `supervisor.rescan_checked` preview was computed
322    /// against.
323    generation: Arc<std::sync::atomic::AtomicU64>,
324}
325
326/// Refusal labels passed to `observe_route_open_refusal` that mean the target
327/// module is not serving right now, and so open or extend an outage in the
328/// route outage tracker. Every one of them is only reachable after the target
329/// was found in the registry, which is what keeps an arbitrary client-chosen
330/// id from ever creating tracker state.
331///
332/// Deliberately absent: `not_registered` and `removed` (the id may be
333/// anything a client sent, and a removed module is gone on purpose),
334/// `protocol_none` (such a module never serves routes, so nothing is out),
335/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
336/// capability and admission-facts refusals (they refuse the caller, not a
337/// module outage), and `relay_reservation_failed` (its code ranges over
338/// capacity limits as well as a vanished connection). Capacity, breaker,
339/// relay-timeout and module-rejection refusals do not pass through that
340/// function at all; the breaker logs its own transitions.
341///
342/// The two not-serving refusals that bypass that function record themselves
343/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
344/// `required_capability_unprovided` is not tracked: the module itself is up,
345/// and the outage belongs to the missing provider.
346const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
347    "reloading",
348    "supervisor_not_live",
349    "registration_not_active",
350    "no_forwarding_connection",
351    "relay_send_failed",
352];
353
354/// Real channel-0 control handler for subc itself.
355#[derive(Clone)]
356pub struct ControlHandler {
357    registry: Arc<Registry>,
358    activity: Option<crate::activity::ActivityLedger>,
359    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
360    forwarding: Arc<ForwardingTable>,
361    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
362    supervisor: SupervisorHandle,
363    subc_capabilities: Arc<[String]>,
364    /// Daemon-wide route.bind relay budget. Used as the fallback when the
365    /// target module has no per-module override in
366    /// `route_bind_relay_timeouts`.
367    route_bind_relay_timeout: Duration,
368    /// Per-module route.bind relay budget overrides, keyed by module id. When
369    /// `handle_route_open` resolves the deadline for a target module, a
370    /// per-module entry wins over the daemon-wide value above.
371    route_bind_relay_timeouts: BTreeMap<String, Duration>,
372    /// Per-target-module bind-relay breaker state. Shared with the forwarding
373    /// table, which is where a new module connection resets it.
374    route_bind_breakers: RouteBindBreakers,
375    /// Live relay admissions keyed by target module. Shared through the
376    /// forwarding table so cloned or separately built handlers enforce one cap.
377    route_bind_concurrency: RouteBindConcurrency,
378    /// Start and end of each module's not-serving period as seen by
379    /// `route.open`, so an outage gets one line at each edge instead of only
380    /// the per-refusal INFO lines. Taken from the forwarding table, so every
381    /// handler built over one table shares it.
382    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
383    /// Consecutive relay timeouts that open a module's breaker.
384    route_bind_breaker_threshold: u32,
385    /// How long a breaker stays open before one probe is admitted.
386    route_bind_breaker_cooldown: Duration,
387    health_probe_timeout: Duration,
388    /// Central storage policy. When set, each registering module receives its
389    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
390    storage_config: Option<crate::daemon_config::StorageConfig>,
391    /// The machine id established at boot, served on every HELLO_ACK and on
392    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
393    /// changes the file, never this value. `None` serves no id.
394    machine_id: Option<crate::machine_id::MachineId>,
395    admission_facts_carrier_module_id: Option<String>,
396    admission_facts_targets: Option<Vec<String>>,
397    /// Scope records with their sync authorities and tombstones; see
398    /// `crate::scopes`. Shared by clones of this handler, so every connection
399    /// reads and writes one table.
400    scopes: Arc<RwLock<ScopeTable>>,
401    /// Wakes the expiry loop when an accepted scope change may have added a
402    /// deadline, so the loop can wait without ticking while no live scope has
403    /// one. A permit is stored when nothing waits, so no wake is lost.
404    scope_deadline_added: Arc<tokio::sync::Notify>,
405    /// The configured `scope_authority_owners`, kept so a rescan can report a
406    /// changed value as needing a daemon restart; rescan never applies it.
407    scope_authority_owners: Vec<String>,
408    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
409    privileged_operator_prompt: PrivilegedOperatorPrompt,
410    /// The local and peer address of each accepted TCP client connection, read
411    /// from the daemon's own socket, so a privileged request can name its caller.
412    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
413    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
414    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
415    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
416    pub(crate) operator_apps:
417        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
418    /// One launch entry per app, behind a leaf lock: no other lock is taken
419    /// while it is held. Entries are memory-only, so a daemon or
420    /// installed-service restart drops them and a relaunched daemon never
421    /// mistakes an older process for its own launch.
422    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
423    /// Serializes `app.open` launches: the liveness check in the decision and
424    /// the later publish must not interleave across concurrent calls, or two
425    /// calls could both launch and the second publish would orphan the first
426    /// process. Refusals and `already_running` answers share the same lock;
427    /// `app.open` is rare, so the cost is negligible.
428    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
429    /// Test-only Windows signature and flash hooks. Production builds carry
430    /// neither: with the publisher unset, the production table never reaches
431    /// verification, and the default attention hook only logs.
432    #[allow(dead_code)]
433    #[cfg(windows)]
434    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
435    #[allow(dead_code)]
436    #[cfg(windows)]
437    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
438    /// Test-only tracking fault hook: makes opening the tracking handle on
439    /// the launched pid fail, so a test can prove an untrackable launch is
440    /// stopped rather than left running.
441    #[allow(dead_code)]
442    #[cfg(windows)]
443    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
444    /// Test-only count of Windows spawn attempts, so a test can prove a
445    /// refused launch started no process. Unit tests only.
446    #[cfg(all(test, windows))]
447    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
448    #[cfg(test)]
449    caller_lookup: Option<TestCallerLookup>,
450    /// The launch nonce each module connection presented at HELLO, which is how
451    /// a `scope.sync` is matched to the owner's current launch.
452    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
453    rescan: Option<SupervisorRescanContext>,
454    connected_clients: ConnectedClients,
455    counters: DaemonCounters,
456    /// Why this daemon serves no local endpoint, when binding one failed. `None`
457    /// when the endpoint is served, and also on a handler that was never told.
458    local_endpoint_unavailable: Option<String>,
459    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
460    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
461    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
462    /// their usual severity (see [`log_requirement_events`]). A handler built
463    /// without one has a flag that is never set.
464    daemon_shutdown: DaemonShutdownFlag,
465    daemon_provenance: DaemonProvenanceFacts,
466    #[cfg(test)]
467    control_dispatch_delay: Option<Duration>,
468    #[cfg(test)]
469    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
470    /// Holds `route.open` after the route.bind is queued to the module and
471    /// before it is marked enqueued, so a test can make the module's reply (or
472    /// a client teardown) land exactly inside that window.
473    #[cfg(test)]
474    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
475    /// Holds a HELLO after its registry entry is reserved and before its
476    /// forwarding endpoint is installed, so a test can read the catalog and
477    /// open routes while the module is registered but not yet routable. A
478    /// blocking barrier because the HELLO handler is synchronous; the test
479    /// runs the HELLO on its own thread.
480    #[cfg(test)]
481    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
482}
483
484impl fmt::Debug for ControlHandler {
485    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
486        f.debug_struct("ControlHandler")
487            .field("registry", &self.registry)
488            .field("forwarding", &self.forwarding)
489            .field("process_liveness", &self.process_liveness.is_some())
490            .field("supervisor", &self.supervisor)
491            .field("subc_capabilities", &self.subc_capabilities)
492            .finish()
493    }
494}
495
496struct RouteOpenRequest {
497    target: RouteTarget,
498    identity: BindIdentity,
499    consumer_identity: Option<ConsumerIdentity>,
500    consumer_capabilities: Option<Vec<String>>,
501    role_versions: Option<BTreeMap<String, String>>,
502    admission_facts: Option<serde_json::Value>,
503    scope: Option<ScopeSelector>,
504}
505
506#[derive(Clone, Copy)]
507enum RouteOpenPhase {
508    Admission,
509    ProjectRoot,
510    EgressReservation,
511    BindSend,
512    BindAckWait,
513}
514
515/// Store elapsed time for each route-open phase, including attempts that end
516/// in an early refusal or backend error. Admission also includes rechecks and
517/// commit bookkeeping performed between the other timed phases.
518struct RouteOpenTimings {
519    module_id: String,
520    principal: String,
521    admission: Duration,
522    project_root: Duration,
523    egress_reservation: Duration,
524    bind_send: Duration,
525    bind_ack_wait: Duration,
526    phase: Option<RouteOpenPhase>,
527    phase_started_at: Instant,
528}
529
530impl RouteOpenTimings {
531    fn new() -> Self {
532        Self {
533            module_id: String::new(),
534            // Refusals before attestation must not claim a verified principal.
535            principal: "unresolved".to_string(),
536            admission: Duration::ZERO,
537            project_root: Duration::ZERO,
538            egress_reservation: Duration::ZERO,
539            bind_send: Duration::ZERO,
540            bind_ack_wait: Duration::ZERO,
541            phase: Some(RouteOpenPhase::Admission),
542            phase_started_at: Instant::now(),
543        }
544    }
545
546    fn finish(&mut self) {
547        let elapsed = self.phase_started_at.elapsed();
548        match self.phase.take() {
549            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
550            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
551            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
552            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
553            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
554            None => {}
555        }
556    }
557
558    fn enter(&mut self, phase: RouteOpenPhase) {
559        self.finish();
560        self.phase = Some(phase);
561        self.phase_started_at = Instant::now();
562    }
563}
564
565struct RouteBindReservationGuard {
566    forwarding: Arc<ForwardingTable>,
567    endpoint: ModuleEndpointId,
568    relay_corr: u64,
569    armed: bool,
570}
571
572struct ModuleControlRpcGuard {
573    forwarding: Arc<ForwardingTable>,
574    endpoint: ModuleEndpointId,
575    corr: u64,
576    armed: bool,
577}
578
579impl ModuleControlRpcGuard {
580    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
581        Self {
582            forwarding,
583            endpoint,
584            corr,
585            armed: true,
586        }
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for ModuleControlRpcGuard {
595    fn drop(&mut self) {
596        if self.armed {
597            let _ = self
598                .forwarding
599                .cancel_module_control_rpc(self.endpoint, self.corr);
600        }
601    }
602}
603
604impl RouteBindReservationGuard {
605    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
606        Self {
607            forwarding,
608            endpoint,
609            relay_corr,
610            armed: true,
611        }
612    }
613
614    fn release_and_disarm(&mut self) {
615        if !self.armed {
616            return;
617        }
618        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
619            self.endpoint,
620            self.relay_corr,
621            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
622        ) {
623            send_goodbye_target_best_effort(
624                &self.forwarding.counters(),
625                &target,
626                "canceled route.bind",
627            );
628        }
629        self.armed = false;
630    }
631
632    fn disarm(&mut self) {
633        self.armed = false;
634    }
635}
636
637impl Drop for RouteBindReservationGuard {
638    fn drop(&mut self) {
639        self.release_and_disarm();
640    }
641}
642
643/// Per-target-module circuit breaker around the `route.bind` relay.
644///
645/// The connection reader is serial per connection, so a module whose `on_bind`
646/// sits on the ack blocks every LATER frame on the connections that call it,
647/// including calls to unrelated modules. This does not make any module's bind
648/// fast; it stops the daemon paying the full budget again and again for a
649/// condition it has already observed.
650///
651/// State is keyed by TARGET MODULE and shared by every connection: a wedged
652/// module wedges everyone, so what one connection learned should protect the
653/// rest.
654///
655/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
656/// relay to that module has actually timed out, and is removed again when a
657/// relay is accepted or the module reconnects, so it cannot grow with traffic
658/// or with modules that behave.
659///
660/// # Why a `std` mutex here is not the head-of-line defect again
661///
662/// Acquisition never awaits. The critical section is a hash lookup plus a few
663/// integer updates, with no I/O and no `.await` inside it, so a reader task
664/// cannot be descheduled behind it the way it can behind
665/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
666/// primitive, held for the same kind of work, as the refusal counter this very
667/// path already increments.
668///
669/// It is also NOT on the data-plane splice path: only `route.open` and module
670/// registration touch it, so bound-route frames gain no state check and no
671/// contention.
672#[derive(Debug, Clone, Default)]
673pub(crate) struct RouteBindBreakers {
674    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
675}
676
677#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindConcurrency {
679    modules: Arc<Mutex<HashMap<String, usize>>>,
680}
681
682struct RouteBindConcurrencyGuard {
683    concurrency: RouteBindConcurrency,
684    module_id: String,
685}
686
687impl RouteBindConcurrency {
688    /// Admit without waiting. Waiting here would move the bind stall from the
689    /// module reply to a semaphore and restore reader head-of-line blocking.
690    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
691        let mut modules = self
692            .modules
693            .lock()
694            .expect("route.bind concurrency mutex poisoned");
695        let in_flight = modules.entry(module_id.to_string()).or_default();
696        if *in_flight >= limit {
697            return Err(*in_flight);
698        }
699        *in_flight += 1;
700        Ok(RouteBindConcurrencyGuard {
701            concurrency: self.clone(),
702            module_id: module_id.to_string(),
703        })
704    }
705}
706
707impl Drop for RouteBindConcurrencyGuard {
708    fn drop(&mut self) {
709        let mut modules = self
710            .concurrency
711            .modules
712            .lock()
713            .expect("route.bind concurrency mutex poisoned");
714        let remove = {
715            let in_flight = modules
716                .get_mut(&self.module_id)
717                .expect("admitted route.bind has a concurrency entry");
718            *in_flight -= 1;
719            *in_flight == 0
720        };
721        if remove {
722            modules.remove(&self.module_id);
723        }
724    }
725}
726
727#[derive(Debug, Default)]
728struct ModuleBreakerState {
729    /// Relay timeouts observed with no accepted relay in between.
730    consecutive_timeouts: u32,
731    /// `Some` while the breaker is open: the instant the cooldown expires and
732    /// the next arrival may probe. `None` means closed.
733    cooldown_until: Option<Instant>,
734    /// A half-open probe has been admitted and has not settled yet. This is
735    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
736    /// that read the cooldown, so concurrent opens arriving at the moment the
737    /// cooldown expires cannot all decide that they are the probe.
738    probe_in_flight: Option<Arc<()>>,
739}
740
741/// What the breaker decided for one `route.open`, before any relay work.
742enum RouteBindAdmission<'a> {
743    Admitted {
744        guard: RouteBindBreakerGuard<'a>,
745        /// This open is the single half-open probe, so the transition is worth
746        /// one log line.
747        probe: bool,
748    },
749    Refused {
750        consecutive_timeouts: u32,
751        /// What is left of the cooldown. Zero when the refusal is because the
752        /// one probe is already in flight rather than because the cooldown has
753        /// not elapsed.
754        retry_in: Duration,
755        probe_in_flight: bool,
756    },
757}
758
759/// An outstanding admission, which must be told how its relay settled.
760///
761/// `Drop` settles it as inconclusive, so an early return between admission and
762/// the relay -- or the whole handler being cancelled when the client
763/// disconnects -- releases a half-open probe slot instead of leaving the
764/// breaker wedged half-open with no further probes.
765struct RouteBindBreakerGuard<'a> {
766    breakers: RouteBindBreakers,
767    module_id: &'a str,
768    probe_token: Option<Arc<()>>,
769    settled: bool,
770}
771
772impl RouteBindBreakerGuard<'_> {
773    /// The module answered within the budget and took the bind. THE ONLY
774    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
775    /// breaker, which is a transition worth logging.
776    fn record_accepted(&mut self) -> bool {
777        self.settled = true;
778        self.breakers.record_accepted(self.module_id)
779    }
780
781    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
782    /// COUNTS TOWARD OPENING.
783    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
784        self.settled = true;
785        self.breakers.record_timeout(
786            self.module_id,
787            self.probe_token.as_ref(),
788            threshold,
789            cooldown,
790        )
791    }
792
793    /// Everything else: the module REJECTED the bind, its connection went away
794    /// mid-relay, or the waiter was cancelled.
795    ///
796    /// None of these is evidence that a module is slow, and each already has
797    /// its own refusal with its own code. A module that rejects a bind in
798    /// microseconds is healthy and must never be convicted for it; a module
799    /// that died has said nothing about the module that replaces it. So these
800    /// neither increment nor reset the count -- they only release a probe slot.
801    fn record_inconclusive(&mut self) {
802        self.settled = true;
803        self.breakers
804            .record_inconclusive(self.module_id, self.probe_token.as_ref());
805    }
806}
807
808impl Drop for RouteBindBreakerGuard<'_> {
809    fn drop(&mut self) {
810        if !self.settled {
811            self.breakers
812                .record_inconclusive(self.module_id, self.probe_token.as_ref());
813        }
814    }
815}
816
817/// The breaker moved to open, reported so the caller can log it outside the
818/// lock. Opening is rare and load-bearing; the refusals that follow are
819/// frequent and are counted rather than logged.
820struct BreakerOpened {
821    consecutive_timeouts: u32,
822    /// True when a failed probe re-opened an already-open breaker, which reads
823    /// very differently in a log from a first opening.
824    reopened_after_probe: bool,
825}
826
827impl RouteBindBreakers {
828    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
829        self.modules
830            .lock()
831            .expect("route.bind breaker mutex poisoned")
832    }
833
834    /// Decide whether this `route.open` may attempt its relay. Takes the map
835    /// lock and nothing else, and never awaits.
836    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
837        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
838            probe: probe_token.is_some(),
839            guard: RouteBindBreakerGuard {
840                breakers: self.clone(),
841                module_id,
842                probe_token,
843                settled: false,
844            },
845        };
846
847        let mut modules = self.lock();
848        let Some(state) = modules.get_mut(module_id) else {
849            return admitted(None);
850        };
851        let Some(cooldown_until) = state.cooldown_until else {
852            return admitted(None);
853        };
854        if state.probe_in_flight.is_some() {
855            return RouteBindAdmission::Refused {
856                consecutive_timeouts: state.consecutive_timeouts,
857                retry_in: Duration::ZERO,
858                probe_in_flight: true,
859            };
860        }
861        let now = Instant::now();
862        if now < cooldown_until {
863            return RouteBindAdmission::Refused {
864                consecutive_timeouts: state.consecutive_timeouts,
865                retry_in: cooldown_until - now,
866                probe_in_flight: false,
867            };
868        }
869        let token = Arc::new(());
870        state.probe_in_flight = Some(Arc::clone(&token));
871        admitted(Some(token))
872    }
873
874    fn record_accepted(&self, module_id: &str) -> bool {
875        self.lock()
876            .remove(module_id)
877            .is_some_and(|state| state.cooldown_until.is_some())
878    }
879
880    fn record_timeout(
881        &self,
882        module_id: &str,
883        probe_token: Option<&Arc<()>>,
884        threshold: u32,
885        cooldown: Duration,
886    ) -> Option<BreakerOpened> {
887        let mut modules = self.lock();
888        let state = modules.entry(module_id.to_string()).or_default();
889        let was_open = state.cooldown_until.is_some();
890        let was_probe = Self::owns_probe(state, probe_token);
891        if was_probe {
892            state.probe_in_flight = None;
893        }
894        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
895        if state.consecutive_timeouts < threshold {
896            return None;
897        }
898        state.cooldown_until = Some(Instant::now() + cooldown);
899        Some(BreakerOpened {
900            consecutive_timeouts: state.consecutive_timeouts,
901            reopened_after_probe: was_open && was_probe,
902        })
903    }
904
905    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
906        // A relay can finish after the breaker was reset or after it opened
907        // again and started a new probe. Only the guard whose token matches the
908        // active probe may release it, so a late relay never frees a newer probe.
909        state
910            .probe_in_flight
911            .as_ref()
912            .zip(token)
913            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
914    }
915
916    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
917        if let Some(state) = self.lock().get_mut(module_id) {
918            if Self::owns_probe(state, probe_token) {
919                state.probe_in_flight = None;
920            }
921        }
922    }
923
924    /// Discard what was learned about a module, because the process it was
925    /// learned about is gone. Returns the discarded count when it was non-zero.
926    ///
927    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
928    /// `module_id` is a configuration identity that outlives any particular
929    /// child; what the breaker observed was the process behind the module
930    /// connection of the moment. When a new connection registers under that id
931    /// the verdict's subject no longer exists, so the verdict is stale by
932    /// construction rather than merely likely to be wrong. Keeping it would
933    /// apply a dead process's record to a live one, which is the same defect
934    /// class this breaker exists to stop the daemon committing.
935    ///
936    /// A half-open probe in flight is discarded with the rest: it was a
937    /// question about the old process.
938    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
939        self.lock()
940            .remove(module_id)
941            .map(|state| state.consecutive_timeouts)
942            .filter(|discarded| *discarded > 0)
943    }
944
945    /// Open breakers, for the `server.describe` counters object. `None` when
946    /// none is open, so the key stays absent rather than present-and-empty.
947    ///
948    /// This is the operator's answer to "is this module refusing instantly or
949    /// is it fine?", which look identical from a client that retries and then
950    /// succeeds.
951    fn open_snapshot(&self) -> Option<serde_json::Value> {
952        let now = Instant::now();
953        let modules = self.lock();
954        let open = modules
955            .iter()
956            .filter_map(|(module_id, state)| {
957                let cooldown_until = state.cooldown_until?;
958                Some((
959                    module_id.clone(),
960                    serde_json::json!({
961                        "consecutive_timeouts": state.consecutive_timeouts,
962                        "cooldown_remaining_ms":
963                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
964                        "probe_in_flight": state.probe_in_flight.is_some(),
965                    }),
966                ))
967            })
968            .collect::<serde_json::Map<String, serde_json::Value>>();
969        (!open.is_empty()).then_some(serde_json::Value::Object(open))
970    }
971}
972
973impl ControlHandler {
974    pub fn new(registry: Arc<Registry>) -> Self {
975        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
976    }
977
978    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
979        let counters = forwarding.counters();
980        // Taken from the forwarding table rather than created here, so that the
981        // breaker a `route.open` consults is the same one a module's
982        // registration resets, however many handlers are built over one table.
983        let route_bind_breakers = forwarding.route_bind_breakers();
984        let route_bind_concurrency = forwarding.route_bind_concurrency();
985        let route_outages = forwarding.route_outages();
986        Self {
987            registry,
988            activity: None,
989            activity_streams: Arc::new(Mutex::new(HashMap::new())),
990            forwarding,
991            process_liveness: None,
992            supervisor: SupervisorHandle::new(),
993            subc_capabilities: Arc::from([
994                CAP_MANIFEST_REGISTRATION.to_string(),
995                CAP_CHANNEL_LIFECYCLE.to_string(),
996                CAP_PING_PONG.to_string(),
997                CAP_SESSION_ATTACH.to_string(),
998                CAP_ADMISSION_FACTS_RELAY.to_string(),
999                CAP_SCOPES_V1.to_string(),
1000                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1001                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1002                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1003                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1004                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1005                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1006                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1007                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1008                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1009                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1010                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1011            ]),
1012            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1013            route_bind_relay_timeouts: BTreeMap::new(),
1014            route_bind_breakers,
1015            route_bind_concurrency,
1016            route_outages,
1017            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1018            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1019            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1020            storage_config: None,
1021            machine_id: None,
1022            admission_facts_carrier_module_id: None,
1023            admission_facts_targets: None,
1024            scopes: Arc::new(RwLock::new(ScopeTable::new(
1025                crate::daemon_config::default_scope_authority_owners(),
1026            ))),
1027            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1028            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1029            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1030            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1031            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1032            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1033            operator_app_policy: Default::default(),
1034            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1035            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1036            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1037            #[cfg(windows)]
1038            windows_verify: None,
1039            #[cfg(windows)]
1040            windows_flash: None,
1041            #[cfg(windows)]
1042            windows_track: None,
1043            #[cfg(all(test, windows))]
1044            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1045            #[cfg(test)]
1046            caller_lookup: None,
1047            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1048            rescan: None,
1049            connected_clients: ConnectedClients::new(),
1050            local_endpoint_unavailable: None,
1051            counters,
1052            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1053            daemon_shutdown: DaemonShutdownFlag::default(),
1054            daemon_provenance: DaemonProvenanceFacts::default(),
1055            #[cfg(test)]
1056            control_dispatch_delay: None,
1057            #[cfg(test)]
1058            provenance_probe_override: None,
1059            #[cfg(test)]
1060            route_bind_relay_sent_hold: None,
1061            #[cfg(test)]
1062            hello_registry_to_forwarding_hold: None,
1063        }
1064    }
1065
1066    /// Set the central storage policy: registering modules then receive their
1067    /// resolved storage descriptor in HELLO_ACK.
1068    pub fn with_storage_config(
1069        mut self,
1070        storage_config: Option<crate::daemon_config::StorageConfig>,
1071    ) -> Self {
1072        self.storage_config = storage_config;
1073        self
1074    }
1075
1076    /// Open the ledger on its own writer thread; an unavailable sink does not
1077    /// prevent the daemon from serving routing or supervisor requests.
1078    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1079        self.activity = Some(crate::activity::ActivityLedger::open(
1080            run_dir.join("activity.jsonl"),
1081            incarnation,
1082        ));
1083        self.supervisor.set_activity(self.activity.clone());
1084        self
1085    }
1086
1087    /// Set the machine id served to every registering module (HELLO_ACK) and on
1088    /// `server.describe`.
1089    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1090        self.machine_id = machine_id;
1091        self
1092    }
1093
1094    /// Configure the exact reserved module and target ids permitted to relay
1095    /// opaque admission facts. Config-file loading validates this authority;
1096    /// this builder keeps the same policy available to embedded test daemons.
1097    pub fn with_admission_facts_config(
1098        mut self,
1099        carrier_module_id: Option<String>,
1100        targets: Option<Vec<String>>,
1101    ) -> Self {
1102        self.admission_facts_carrier_module_id = carrier_module_id;
1103        self.admission_facts_targets = targets;
1104        self
1105    }
1106
1107    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1108    /// Replaces the scope table with an empty one under the new list, so call it
1109    /// while building the handler, before any module can sync.
1110    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1111        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1112        self.scope_authority_owners = owners;
1113        self
1114    }
1115
1116    /// Set the startup policy before installing the rescan context or serving requests.
1117    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1118        self.privileged_operator_prompt = mode;
1119        self
1120    }
1121
1122    /// Override the route.bind relay timeout. Used by tests that assert the
1123    /// timeout path so they don't block on the production-safe default.
1124    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1125        self.route_bind_relay_timeout = timeout;
1126        self
1127    }
1128
1129    /// Install per-module route.bind relay budget overrides. A module id
1130    /// listed here wins over the daemon-wide default set via
1131    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1132    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1133    /// the same `Duration` the bind path will use.
1134    pub fn with_route_bind_relay_timeouts(
1135        mut self,
1136        timeouts: impl IntoIterator<Item = (String, Duration)>,
1137    ) -> Self {
1138        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1139        self
1140    }
1141
1142    /// Resolve the route.bind relay budget for a specific target module id.
1143    /// Per-module overrides win; the daemon-wide value (set via
1144    /// `with_route_bind_relay_timeout` or the built-in default) is the
1145    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1146    /// the same resolution `handle_route_open` will use.
1147    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1148        self.route_bind_relay_timeouts
1149            .get(module_id)
1150            .copied()
1151            .unwrap_or(self.route_bind_relay_timeout)
1152    }
1153
1154    /// Override the per-module bind-relay breaker policy.
1155    ///
1156    /// Used by tests, which cannot spend three production budgets opening a
1157    /// breaker or twenty seconds waiting for its cooldown. The production
1158    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1159    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1160    /// reasoning for the numbers.
1161    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1162        self.route_bind_breaker_threshold = threshold.max(1);
1163        self.route_bind_breaker_cooldown = cooldown;
1164        self
1165    }
1166
1167    #[cfg(test)]
1168    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1169        self.health_probe_timeout = timeout;
1170        self
1171    }
1172
1173    #[cfg(test)]
1174    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1175        self.control_dispatch_delay = Some(delay);
1176        self
1177    }
1178
1179    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1180    /// announce it is parked, once to be released. Build the barrier for two
1181    /// parties, the test being the other one.
1182    #[cfg(test)]
1183    pub(crate) fn with_route_bind_relay_sent_hold(
1184        mut self,
1185        hold: Arc<tokio::sync::Barrier>,
1186    ) -> Self {
1187        self.route_bind_relay_sent_hold = Some(hold);
1188        self
1189    }
1190
1191    /// A HELLO waits on `hold` twice between reserving its registry entry and
1192    /// installing its forwarding endpoint: once to announce it is parked, once
1193    /// to be released. Build the barrier for two parties, the test being the
1194    /// other one, and run the HELLO on a thread of its own.
1195    #[cfg(test)]
1196    pub(crate) fn with_hello_registry_to_forwarding_hold(
1197        mut self,
1198        hold: Arc<std::sync::Barrier>,
1199    ) -> Self {
1200        self.hello_registry_to_forwarding_hold = Some(hold);
1201        self
1202    }
1203
1204    #[cfg(test)]
1205    fn hold_between_registry_and_forwarding(&self) {
1206        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1207            hold.wait();
1208            hold.wait();
1209        }
1210    }
1211
1212    pub fn with_process_liveness(
1213        mut self,
1214        process_liveness: Arc<dyn ModuleProcessLiveness>,
1215    ) -> Self {
1216        self.process_liveness = Some(process_liveness);
1217        self
1218    }
1219
1220    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1221        self.supervisor = supervisor;
1222        self.supervisor.set_activity(self.activity.clone());
1223        self
1224    }
1225
1226    pub fn with_daemon_provenance(
1227        mut self,
1228        pid: u32,
1229        started_at_ms: u64,
1230        executable_path: Option<PathBuf>,
1231        build_git_sha: Option<String>,
1232        build_lock_digest: Option<String>,
1233    ) -> Self {
1234        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1235        let process_start_time = process_start_time(pid);
1236        self.daemon_provenance = DaemonProvenanceFacts {
1237            build: DaemonBuildProvenance {
1238                build_git_sha,
1239                build_lock_digest,
1240            },
1241            pid: Some(pid),
1242            started_at_ms: Some(started_at_ms),
1243            start_clock: None,
1244            executable_path,
1245            executable_identity,
1246            process_start_time,
1247            probe: ExecutableIdentityProbe::default(),
1248        };
1249        self
1250    }
1251
1252    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1253        self.daemon_provenance.start_clock = Some(clock);
1254        self
1255    }
1256
1257    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1258    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1259        self.scopes
1260            .write()
1261            .unwrap_or_else(|poisoned| poisoned.into_inner())
1262            .set_wall_clock(Arc::new(clock));
1263        self
1264    }
1265
1266    #[cfg(test)]
1267    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1268        self.provenance_probe_override = Some(result);
1269        self
1270    }
1271
1272    /// Install the configured module set and its reserved capability bindings.
1273    /// Bindings are configuration-scoped and may point at a provider that has not
1274    /// been installed yet, so this does not require the bound module to exist.
1275    pub fn with_capability_config(
1276        self,
1277        modules: impl IntoIterator<Item = (String, bool)>,
1278        reserved_capabilities: BTreeMap<String, String>,
1279    ) -> Self {
1280        self.capability_evaluator
1281            .configure(modules, reserved_capabilities);
1282        self
1283    }
1284
1285    /// Share the supervisor's daemon-shutdown flag with this handler, so
1286    /// requirement changes after the daemon has begun its announced shutdown
1287    /// are not reported as faults.
1288    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1289        self.daemon_shutdown = flag;
1290        self
1291    }
1292
1293    pub fn with_supervisor_rescan(
1294        mut self,
1295        supervisor: Supervisor,
1296        config_path: impl Into<PathBuf>,
1297        configured_port: Option<u16>,
1298    ) -> Self {
1299        self.rescan = Some(SupervisorRescanContext {
1300            supervisor,
1301            config_path: config_path.into(),
1302            configured_port,
1303            storage_config: self.storage_config.clone(),
1304            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1305            admission_facts_targets: self.admission_facts_targets.clone(),
1306            scope_authority_owners: self.scope_authority_owners.clone(),
1307            privileged_operator_prompt: self.privileged_operator_prompt,
1308            generation: Arc::default(),
1309        });
1310        self
1311    }
1312
1313    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1314        self.connected_clients = connected_clients;
1315        self
1316    }
1317
1318    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1319    /// an operator can see a bind failure without reading the daemon log.
1320    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1321        self.local_endpoint_unavailable = reason;
1322        self
1323    }
1324
1325    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1326        Arc::clone(&self.forwarding)
1327    }
1328
1329    pub(crate) fn counters(&self) -> DaemonCounters {
1330        self.counters.clone()
1331    }
1332
1333    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1334    /// requirement event without depending on an operator polling a status command.
1335    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1336        tokio::spawn(async move {
1337            loop {
1338                self.capability_evaluator
1339                    .wait_for_change_or_deadline()
1340                    .await;
1341                self.refresh_capability_requirements();
1342            }
1343        });
1344    }
1345
1346    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1347    /// deadline passed during sleep is enforced on the first tick after wake.
1348    /// While no live scope carries a deadline the loop waits for an accepted
1349    /// scope change instead of ticking, so a daemon without deadlines never
1350    /// wakes for expiry.
1351    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1352        tokio::spawn(async move {
1353            loop {
1354                let has_deadlines = self
1355                    .scopes
1356                    .read()
1357                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1358                    .has_deadlines();
1359                if !has_deadlines {
1360                    self.scope_deadline_added.notified().await;
1361                    continue;
1362                }
1363                tokio::time::sleep(Duration::from_secs(1)).await;
1364                if let Err(error) = self.sweep_expired_scopes() {
1365                    warn!(%error, "scope expiry sweep failed");
1366                }
1367            }
1368        });
1369    }
1370
1371    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1372        let mut table = self
1373            .scopes
1374            .write()
1375            .unwrap_or_else(|poisoned| poisoned.into_inner());
1376        let swept = table.sweep_expired();
1377        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1378        drop(table);
1379        self.close_scope_drained_routes(drained);
1380        Ok(())
1381    }
1382
1383    fn publish_scope_changes(
1384        &self,
1385        changes: &[crate::scopes::ScopeTagChange],
1386        expired: &[crate::scopes::ScopeExpired],
1387    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1388        if changes.is_empty() {
1389            return Ok(Vec::new());
1390        }
1391        let drained = self
1392            .forwarding
1393            .publish_scope_changes(changes)
1394            .map_err(RouterError::Forwarding)?;
1395        let mut counts = HashMap::new();
1396        for route in &drained {
1397            *counts
1398                .entry((
1399                    route.scope.owner.as_str(),
1400                    route.scope.scope_ref.as_str(),
1401                    route.scope.tag.scope_epoch,
1402                ))
1403                .or_insert(0usize) += 1;
1404        }
1405        for scope in expired {
1406            let routes_closed = counts
1407                .get(&(
1408                    scope.owner.as_str(),
1409                    scope.scope_ref.as_str(),
1410                    scope.scope_epoch,
1411                ))
1412                .copied()
1413                .unwrap_or(0);
1414            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1415                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1416                routes_closed, "scope expired");
1417        }
1418        Ok(drained)
1419    }
1420
1421    fn runtime_capability_snapshot(
1422        &self,
1423    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1424        let runtime = self
1425            .supervisor
1426            .list()
1427            .into_iter()
1428            .map(|module| {
1429                let status = module.status().map_err(|err| {
1430                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1431                })?;
1432                Ok(RuntimeModule {
1433                    module_id: status.module_id,
1434                    state: status.state,
1435                    enabled: status.enabled,
1436                })
1437            })
1438            .collect::<Result<Vec<_>, RouterError>>()?;
1439        // Include every registration, published or not. The capability
1440        // evaluator counts a module as providing its capabilities as soon as it
1441        // has registered, even before `catalog.list` shows it, so a HELLO in
1442        // progress does not change other modules' required-capability
1443        // verdicts. See `Registry::list_registered_modules`.
1444        let registrations = self.registry.list_registered_modules().map_err(|err| {
1445            RouterError::backend(
1446                0,
1447                0,
1448                format!("failed to list capability registrations: {err}"),
1449            )
1450        })?;
1451        let registrations = registrations
1452            .into_iter()
1453            .map(|registration| RegisteredModule {
1454                module_id: registration.manifest.module_id,
1455                module_version: registration.manifest.module_version,
1456                capabilities: registration.manifest.capabilities,
1457            })
1458            .collect();
1459        Ok((runtime, registrations))
1460    }
1461
1462    /// The capability side effects of a module becoming the active registration
1463    /// for its id: cache its manifest (logging how it differs from the cached
1464    /// one), run the deny census when its declarations call for one, and recompute the
1465    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1466    /// candidate's does not, and the supervisor does it at promotion instead,
1467    /// through [`crate::supervise::SwapPromotionObserver`].
1468    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1469        let cached_registration = RegisteredModule {
1470            module_id: registration.manifest.module_id.clone(),
1471            module_version: registration.manifest.module_version.clone(),
1472            capabilities: registration.manifest.capabilities.clone(),
1473        };
1474        let change = self.capability_evaluator.record_hello(&cached_registration);
1475        log_manifest_change(&cached_registration.module_id, &change);
1476        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1477            self.enforce_capability_denies();
1478        }
1479        self.refresh_capability_requirements();
1480    }
1481
1482    /// Point the shared supervisor handle at this handler for swap promotions.
1483    /// Called wherever a handler is put behind the `Arc` the router serves, so
1484    /// it can be held weakly.
1485    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1486        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1487            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1488        self.supervisor.set_swap_promotion_observer(observer);
1489    }
1490
1491    pub fn refresh_capability_requirements(&self) {
1492        match self.runtime_capability_snapshot() {
1493            Ok((runtime, registrations)) => {
1494                log_requirement_events(
1495                    self.capability_evaluator
1496                        .evaluate_now(&runtime, &registrations),
1497                    &self.daemon_shutdown,
1498                );
1499            }
1500            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1501        }
1502    }
1503
1504    /// Reconcile only live, attested route bindings after a capability deny edge
1505    /// or target claim was added. This is deliberately a control-plane census:
1506    /// the opaque forwarding hot path must not grow a per-frame capability check.
1507    fn enforce_capability_denies(&self) {
1508        let registrations = match self.registry.list_registered_modules() {
1509            Ok(snapshot) => snapshot,
1510            Err(err) => {
1511                warn!(error = %err, "failed to read registrations for capability deny census");
1512                return;
1513            }
1514        };
1515        let manifests = registrations
1516            .into_iter()
1517            .map(|registration| {
1518                (
1519                    registration.manifest.module_id.clone(),
1520                    registration.manifest,
1521                )
1522            })
1523            .collect::<BTreeMap<_, _>>();
1524        let census = match self.forwarding.route_census(None) {
1525            Ok(census) => census,
1526            Err(err) => {
1527                warn!(error = %err, "failed to read route census for capability deny enforcement");
1528                return;
1529            }
1530        };
1531
1532        for (target_module_id, routes) in census {
1533            let Some(target_manifest) = manifests.get(&target_module_id) else {
1534                continue;
1535            };
1536            let mut closed_routes = Vec::new();
1537            let mut goodbyes = Vec::new();
1538            for route in routes {
1539                let Principal::Reserved {
1540                    module_id: opening_module_id,
1541                } = &route.principal
1542                else {
1543                    continue;
1544                };
1545                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1546                    continue;
1547                };
1548                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1549                    continue;
1550                };
1551
1552                match self.forwarding.release_client_route(
1553                    route.goodbye_target.connection_id,
1554                    route.goodbye_target.channel,
1555                    route.goodbye_target.epoch,
1556                ) {
1557                    Ok(RouteRelease::Removed(module_goodbye)) => {
1558                        warn!(
1559                            opening_module_id,
1560                            target_module_id,
1561                            capability,
1562                            "force-closing route because an attested capability deny edge now matches"
1563                        );
1564                        goodbyes.push(module_goodbye);
1565                        goodbyes.push(route.goodbye_target.clone());
1566                        closed_routes.push(route);
1567                    }
1568                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1569                    Err(err) => warn!(
1570                        opening_module_id,
1571                        target_module_id,
1572                        capability,
1573                        error = %err,
1574                        "failed to force-close capability-denied route"
1575                    ),
1576                }
1577            }
1578
1579            if closed_routes.is_empty() {
1580                continue;
1581            }
1582            send_route_control_pushes(
1583                &self.forwarding,
1584                closed_routes,
1585                ClientControlPush::RouteClosed {
1586                    module_id: target_module_id,
1587                    channels: Vec::new(),
1588                    reason: RouteCloseReason::CapabilityDenied,
1589                    drained: false,
1590                    abandoned: 0,
1591                    excluded_subscriptions: 0,
1592                    terminal: Some(false),
1593                },
1594            );
1595            // route.closed records the reason; the route GOODBYE ends the
1596            // client's handle and settles any calls still pending on it.
1597            self.emit_route_goodbyes(goodbyes);
1598        }
1599    }
1600
1601    /// Why a registered module is not accepting new route binds, or `None` when
1602    /// it is. This is the module's effective readiness: its declared readiness
1603    /// first, then every `need: required` capability it declares evaluating to
1604    /// `provided`. `route.open` and `catalog.list` both read it here so the
1605    /// catalog never reports a module routable that `route.open` would refuse.
1606    fn not_ready_reason(
1607        &self,
1608        registration: &crate::registry::ModuleRegistration,
1609    ) -> Option<NotReadyReason> {
1610        if !registration.ready {
1611            return Some(NotReadyReason {
1612                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1613                capability: None,
1614            });
1615        }
1616        self.first_unprovided_required_capability(registration)
1617            .map(|capability| NotReadyReason {
1618                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1619                capability: Some(capability),
1620            })
1621    }
1622
1623    /// The lexicographically first capability this registration declares
1624    /// `need: required` whose evaluator verdict is not `provided`.
1625    ///
1626    /// The verdicts are the capability evaluator's own; nothing here decides
1627    /// what "provided" means. The evaluator counts a capability provided as
1628    /// soon as a module claiming it has REGISTERED, not once that module is
1629    /// ready. That distinction is what keeps two modules that require each
1630    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1631    /// is ready", each would wait for the other to become ready first and
1632    /// neither ever would. Do not tighten it to readiness.
1633    ///
1634    /// A required capability with no verdict at all means this registration's
1635    /// HELLO or catalog.update landed after the last recompute; recompute once
1636    /// rather than let a missing verdict read as either answer. If it is still
1637    /// missing (the recompute itself failed) the capability counts as
1638    /// unprovided: the refusal is retryable, and routing a module whose
1639    /// required provider is unknown is the outcome this check exists to stop.
1640    fn first_unprovided_required_capability(
1641        &self,
1642        registration: &crate::registry::ModuleRegistration,
1643    ) -> Option<String> {
1644        let required = registration
1645            .manifest
1646            .capabilities
1647            .iter()
1648            .flat_map(|declarations| declarations.requires.iter())
1649            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1650            .map(|requirement| requirement.capability.as_str())
1651            .collect::<BTreeSet<_>>();
1652        if required.is_empty() {
1653            return None;
1654        }
1655        let module_id = registration.manifest.module_id.as_str();
1656        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1657        if required
1658            .iter()
1659            .any(|capability| verdict(capability).is_none())
1660        {
1661            self.refresh_capability_requirements();
1662        }
1663        required
1664            .into_iter()
1665            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1666            .map(str::to_string)
1667    }
1668
1669    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1670        self.capability_evaluator
1671            .statuses()
1672            .into_iter()
1673            .map(capability_requirement_status)
1674            .collect()
1675    }
1676
1677    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1678    /// registration-release watch. The signal is what the supervisor waits on
1679    /// before spawning a replacement, so it must only fire once forwarding
1680    /// teardown is also done (see [`Self::cleanup_connection`] /
1681    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1682    /// admitted but its forwarding endpoint could not be installed.
1683    fn deregister_connection(
1684        &self,
1685        connection_id: ConnectionId,
1686        reason: RegistrationEndReason,
1687    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1688        self.registry
1689            .deregister_connection_with_reason(connection_id, reason)
1690    }
1691
1692    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1693        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1694            return None;
1695        }
1696        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1697            parse_client_control_request(&frame.body)
1698        else {
1699            return None;
1700        };
1701        Some(target_module_id(&target).to_string())
1702    }
1703
1704    pub(crate) fn route_open_capacity_refusal(
1705        &self,
1706        ctx: &RouteCtx,
1707        frame: &Frame,
1708        target_module_id: &str,
1709        in_flight: usize,
1710        limit: usize,
1711    ) -> Result<Frame, RouterError> {
1712        // Sent before the request reaches `handle_route_open`, so the caller
1713        // fields come from the raw frame here.
1714        let caller = self.route_open_caller_from_frame(frame);
1715        self.route_open_admission_refusal_frame(
1716            ctx,
1717            &caller,
1718            frame,
1719            target_module_id,
1720            "open_admission_full",
1721            (in_flight, limit),
1722            format!(
1723                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1724            ),
1725        )
1726    }
1727
1728    fn route_open_target_capacity_refusal(
1729        &self,
1730        ctx: &RouteCtx,
1731        caller: &RouteOpenCaller,
1732        frame: &Frame,
1733        target_module_id: &str,
1734        in_flight: usize,
1735    ) -> Result<Frame, RouterError> {
1736        self.route_open_admission_refusal_frame(
1737            ctx,
1738            caller,
1739            frame,
1740            target_module_id,
1741            "target_binds_full",
1742            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1743            format!(
1744                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1745            ),
1746        )
1747    }
1748
1749    /// Admission pressure clears as existing binds settle, so its refusal must
1750    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1751    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1752    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1753    /// cannot currently reach its target; `module_timeout` would falsely claim
1754    /// that a wait expired. A new, cleaner code would be terminal to deployed
1755    /// clients, so it requires a client-tolerance rollout before daemon emission.
1756    #[allow(clippy::too_many_arguments)]
1757    fn route_open_admission_refusal_frame(
1758        &self,
1759        ctx: &RouteCtx,
1760        caller: &RouteOpenCaller,
1761        frame: &Frame,
1762        target_module_id: &str,
1763        reason: &'static str,
1764        (in_flight, limit): (usize, usize),
1765        message: impl Into<String>,
1766    ) -> Result<Frame, RouterError> {
1767        let code = error_codes::TARGET_UNAVAILABLE;
1768        self.counters.increment_route_open_refused(code);
1769        info!(
1770            target: "control",
1771            code,
1772            reason,
1773            module_id = ?target_module_id,
1774            connection_id = ctx.connection_id.get(),
1775            principal = %caller.principal,
1776            scope = %caller.scope,
1777            harness = %caller.harness,
1778            in_flight,
1779            limit,
1780            "route.open refused"
1781        );
1782        control_error_frame(frame, code, message.into())
1783    }
1784
1785    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1786    ///
1787    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1788    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1789    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1790    #[cfg(test)]
1791    pub fn handle_control(
1792        &self,
1793        connection_id: ConnectionId,
1794        frame: Frame,
1795    ) -> Result<Vec<Frame>, RouterError> {
1796        match frame.header.ty {
1797            FrameType::Ping => Ok(vec![pong(&frame)?]),
1798            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1799            FrameType::Goodbye => self.handle_goodbye(connection_id),
1800            ty => Ok(vec![control_error_frame(
1801                &frame,
1802                "unsupported_control_frame",
1803                format!("unsupported channel-0 frame {ty:?}"),
1804            )?]),
1805        }
1806    }
1807
1808    pub async fn handle_control_frame(
1809        &self,
1810        ctx: &RouteCtx,
1811        frame: Frame,
1812    ) -> Result<Vec<Frame>, RouterError> {
1813        self.handle_control_frame_timed(ctx, frame, None).await
1814    }
1815
1816    #[cfg(any(test, feature = "test-support"))]
1817    pub fn with_test_operator_app_policy(
1818        mut self,
1819        policy: crate::operator_apps::TestOperatorAppPolicy,
1820    ) -> Self {
1821        self.operator_app_policy = policy.into();
1822        self
1823    }
1824
1825    /// Inject the Windows signature check for tests. The injected check
1826    /// answers the verified version string or the rejection reason; the
1827    /// daemon enforces the floor itself on the returned version.
1828    #[cfg(all(windows, any(test, feature = "test-support")))]
1829    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1830        self.windows_verify = Some(verify);
1831        self
1832    }
1833
1834    /// Record which pids the daemon would flash for `already_running`.
1835    #[cfg(all(windows, any(test, feature = "test-support")))]
1836    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1837        self.windows_flash = Some(flash);
1838        self
1839    }
1840
1841    /// Inject a tracking failure for tests: makes `track_launched_process`
1842    /// fail so a test can prove a launch that cannot be tracked is stopped,
1843    /// never left running.
1844    #[cfg(all(windows, any(test, feature = "test-support")))]
1845    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1846        self.windows_track = Some(track);
1847        self
1848    }
1849
1850    pub(crate) fn record_captured_peer(
1851        &self,
1852        connection: ConnectionId,
1853        peer: subc_os::local_peer::CapturedPeer,
1854    ) {
1855        self.record_observed_peer(connection, peer.observation().clone());
1856        #[cfg(windows)]
1857        if let Some(stamp) = self.stamp_windows_launch(&peer) {
1858            self.operator_apps
1859                .lock()
1860                .unwrap_or_else(|p| p.into_inner())
1861                .insert(connection, stamp);
1862        }
1863        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1864            self.operator_apps
1865                .lock()
1866                .unwrap_or_else(|p| p.into_inner())
1867                .insert(connection, stamp);
1868        }
1869    }
1870
1871    pub(crate) fn record_observed_peer(
1872        &self,
1873        connection: ConnectionId,
1874        peer: subc_os::local_peer::ObservedPeer,
1875    ) {
1876        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1877        let peer = match peer {
1878            OsPeer::Unobserved => ObservedPeer::unobserved(),
1879            OsPeer::Observed(peer) => {
1880                let observed = match peer.user {
1881                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1882                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1883                };
1884                observed
1885                    .with_exe(
1886                        peer.exe
1887                            .and_then(|path| path.into_os_string().into_string().ok()),
1888                    )
1889                    .with_signing_id(peer.signing_id)
1890            }
1891        };
1892        self.observed_peers
1893            .lock()
1894            .unwrap_or_else(|p| p.into_inner())
1895            .insert(connection, peer);
1896    }
1897
1898    pub(crate) fn record_caller_endpoints(
1899        &self,
1900        connection: ConnectionId,
1901        endpoints: Option<(SocketAddr, SocketAddr)>,
1902    ) {
1903        let mut callers = self
1904            .caller_endpoints
1905            .lock()
1906            .unwrap_or_else(|p| p.into_inner());
1907        callers.remove(&connection);
1908        if let Some(endpoints) = endpoints {
1909            callers.insert(connection, endpoints);
1910        }
1911    }
1912
1913    #[cfg(test)]
1914    pub(crate) fn with_caller_lookup_for_test(
1915        mut self,
1916        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1917    ) -> Self {
1918        self.caller_lookup = Some(Arc::new(lookup));
1919        self
1920    }
1921
1922    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1923        let endpoints = self
1924            .caller_endpoints
1925            .lock()
1926            .unwrap_or_else(|p| p.into_inner())
1927            .get(&connection)
1928            .copied();
1929        #[cfg(test)]
1930        let lookup = self.caller_lookup.clone();
1931        tokio::task::spawn_blocking(move || {
1932            #[cfg(test)]
1933            if let Some(lookup) = lookup {
1934                return lookup(endpoints);
1935            }
1936            endpoints
1937                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1938                .unwrap_or_default()
1939        })
1940        .await
1941        .unwrap_or_default()
1942    }
1943
1944    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1945        frame.header.channel == 0
1946            && frame.header.ty == FrameType::Request
1947            && self
1948                .forwarding
1949                .module_endpoint_for_connection(ctx.connection_id)
1950                .is_ok_and(|endpoint| endpoint.is_none())
1951            && parse_client_control_request(&frame.body)
1952                .ok()
1953                .is_some_and(|request| privileged_operation(&request).is_some())
1954    }
1955
1956    pub(crate) async fn handle_control_frame_timed(
1957        &self,
1958        ctx: &RouteCtx,
1959        frame: Frame,
1960        dispatch_started_at: Option<StdInstant>,
1961    ) -> Result<Vec<Frame>, RouterError> {
1962        match frame.header.ty {
1963            FrameType::Ping => Ok(vec![pong(&frame)?]),
1964            FrameType::Hello => {
1965                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1966            }
1967            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1968            FrameType::Cancel => {
1969                // A Cancel on channel 0 names a waiting operator.confirm or a
1970                // subscription. Successful cancellation has no ordinary reply.
1971                if self
1972                    .forwarding
1973                    .operator_confirms()
1974                    .cancel(ctx.connection_id, frame.header.corr)
1975                    || self
1976                        .supervisor
1977                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1978                    || self.cancel_activity_subscription(ctx, &frame)?
1979                {
1980                    Ok(Vec::new())
1981                } else {
1982                    Ok(vec![control_error_frame(
1983                        &frame,
1984                        "unknown_subscription",
1985                        "no subscription has this correlation id",
1986                    )?])
1987                }
1988            }
1989            FrameType::Request => {
1990                // This additive operation is not part of the existing exhaustive
1991                // module-control enum. Probe the op before decoding that enum.
1992                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1993                if op
1994                    .as_ref()
1995                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1996                {
1997                    return self.handle_activity_record(ctx, frame).await;
1998                }
1999                if op.as_ref().is_some_and(|probe| {
2000                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2001                }) {
2002                    return self.handle_activity_read(ctx, frame).await;
2003                }
2004                if op
2005                    .as_ref()
2006                    .is_some_and(|probe| probe.op == "operator.confirm")
2007                {
2008                    return self.handle_operator_confirm(ctx, frame);
2009                }
2010                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2011                    return self.handle_app_open(ctx, frame).await;
2012                }
2013                if op
2014                    .as_ref()
2015                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2016                {
2017                    return self.handle_scope_served_by(ctx.connection_id, frame);
2018                }
2019                if self
2020                    .forwarding
2021                    .module_endpoint_for_connection(ctx.connection_id)
2022                    .map_err(RouterError::Forwarding)?
2023                    .is_some()
2024                {
2025                    if !is_known_module_request_op(&frame.body) {
2026                        return Ok(vec![control_error_frame(
2027                            &frame,
2028                            "unsupported_control_frame",
2029                            "module-originated channel-0 REQUEST is not supported",
2030                        )?]);
2031                    }
2032                    let request = match parse_module_control_request_from_module(&frame.body) {
2033                        Ok(request) => request,
2034                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2035                            return Ok(vec![control_error_frame(
2036                                &frame,
2037                                "unsupported_control_frame",
2038                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2039                            )?])
2040                        }
2041                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2042                            return Ok(vec![control_error_frame(
2043                                &frame,
2044                                "invalid_control_body",
2045                                format!("malformed module control body: {err}"),
2046                            )?])
2047                        }
2048                    };
2049                    let op = module_control_request_op(&request);
2050                    let corr = frame.header.corr;
2051                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2052                    let result =
2053                        self.handle_module_control_request(ctx.connection_id, frame, request);
2054                    log_slow_control_dispatch(
2055                        dispatch_started_at,
2056                        op,
2057                        ctx.connection_id,
2058                        corr,
2059                        None,
2060                    );
2061                    return result;
2062                }
2063
2064                if is_known_module_request_op(&frame.body) {
2065                    return Ok(vec![control_error_frame(
2066                        &frame,
2067                        "not_registered",
2068                        "catalog.update requires an active module registration owned by this connection",
2069                    )?]);
2070                }
2071
2072                let request = match parse_client_control_request(&frame.body) {
2073                    Ok(request) => request,
2074                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2075                        return Ok(vec![control_error_frame(
2076                            &frame,
2077                            "unknown_control_op",
2078                            format!("unknown client control op: {err}"),
2079                        )?])
2080                    }
2081                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2082                        return Ok(vec![control_error_frame(
2083                            &frame,
2084                            "invalid_control_body",
2085                            format!("malformed client control body: {err}"),
2086                        )?])
2087                    }
2088                };
2089                let op = client_control_request_op(&request);
2090                let corr = frame.header.corr;
2091                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2092                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2093                #[cfg(test)]
2094                if let Some(delay) = self.control_dispatch_delay {
2095                    tokio::time::sleep(delay).await;
2096                }
2097                let result = self
2098                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2099                    .await;
2100                if let Some(timings) = route_open_timings.as_mut() {
2101                    timings.finish();
2102                }
2103                log_slow_control_dispatch(
2104                    dispatch_started_at,
2105                    op,
2106                    ctx.connection_id,
2107                    corr,
2108                    route_open_timings.as_ref(),
2109                );
2110                result
2111            }
2112            FrameType::Push => {
2113                let Some(endpoint) = self
2114                    .forwarding
2115                    .module_endpoint_for_connection(ctx.connection_id)
2116                    .map_err(RouterError::Forwarding)?
2117                else {
2118                    return Ok(vec![control_error_frame(
2119                        &frame,
2120                        "unsupported_control_frame",
2121                        "client-originated channel-0 PUSH is not supported",
2122                    )?]);
2123                };
2124                self.handle_status_update(endpoint, frame)
2125            }
2126            FrameType::Response | FrameType::Error
2127                if self
2128                    .forwarding
2129                    .module_endpoint_for_connection(ctx.connection_id)
2130                    .map_err(RouterError::Forwarding)?
2131                    .is_some() =>
2132            {
2133                self.handle_module_relay_response(ctx.connection_id, frame)
2134            }
2135            ty => Ok(vec![control_error_frame(
2136                &frame,
2137                "unsupported_control_frame",
2138                format!("unsupported channel-0 frame {ty:?}"),
2139            )?]),
2140        }
2141    }
2142
2143    pub fn cleanup_connection(
2144        &self,
2145        connection_id: ConnectionId,
2146    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2147        self.caller_endpoints
2148            .lock()
2149            .unwrap_or_else(|p| p.into_inner())
2150            .remove(&connection_id);
2151        self.build_class_warned
2152            .lock()
2153            .unwrap_or_else(|p| p.into_inner())
2154            .remove(&connection_id);
2155        self.observed_peers
2156            .lock()
2157            .unwrap_or_else(|p| p.into_inner())
2158            .remove(&connection_id);
2159        self.operator_apps
2160            .lock()
2161            .unwrap_or_else(|p| p.into_inner())
2162            .remove(&connection_id);
2163        self.forwarding
2164            .operator_confirms()
2165            .connection_closed(connection_id);
2166        self.cleanup_connection_with_end_reason(
2167            connection_id,
2168            RegistrationEndReason::ConnectionClosed,
2169        )
2170    }
2171
2172    fn cleanup_connection_with_end_reason(
2173        &self,
2174        connection_id: ConnectionId,
2175        requested_reason: RegistrationEndReason,
2176    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2177        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2178            self.registry
2179                .get_module_by_connection(connection_id)?
2180                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2181                .and_then(|module| module.registration_end_reason().ok().flatten())
2182                .unwrap_or(requested_reason)
2183        } else {
2184            requested_reason
2185        };
2186        let crash_closed = self
2187            .registry
2188            .get_module_by_connection(connection_id)?
2189            .and_then(|registration| {
2190                self.forwarding
2191                    .module_endpoint_for_connection(connection_id)
2192                    .ok()
2193                    .flatten()
2194                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2195                    .map(|routes| (registration.manifest.module_id, routes))
2196            });
2197        let crash_closed = crash_closed.map(|(module_id, routes)| {
2198            let terminal = match self.supervisor.get(&module_id) {
2199                None => false,
2200                Some(module) => match module.will_recover_after_connection_loss() {
2201                    Ok(will_recover) => !will_recover,
2202                    Err(err) => {
2203                        warn!(
2204                            %module_id,
2205                            error = %err,
2206                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2207                        );
2208                        false
2209                    }
2210                },
2211            };
2212            // The forwarding table gates all providers at the start of daemon
2213            // shutdown, before their connections are closed. An ordinary
2214            // module disconnect still reports crash if that gate is not set.
2215            let (reason, drained) = match self
2216                .forwarding
2217                .daemon_drain_result_for_connection(connection_id)
2218            {
2219                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2220                Ok(None) => (RouteCloseReason::Crash, false),
2221                Err(err) => {
2222                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2223                    (RouteCloseReason::Crash, false)
2224                }
2225            };
2226            (module_id, routes, reason, drained, terminal)
2227        });
2228        let registrations = self.deregister_connection(connection_id, end_reason);
2229        let cleanup = if crash_closed.is_some() {
2230            self.forwarding.cleanup_connection_counted(connection_id)
2231        } else {
2232            // A module connection's teardown needs the count of abandoned
2233            // route.bind relays for its route.closed notice below. Any other
2234            // connection, such as a client's, sends no such notice and needs
2235            // only its routes released, so it uses the route-only wrapper and
2236            // reports zero.
2237            self.forwarding
2238                .cleanup_connection(connection_id)
2239                .map(|released| crate::forwarding::ConnectionCleanup {
2240                    released,
2241                    abandoned_relays: 0,
2242                })
2243        };
2244        // The route.closed push waits for forwarding teardown because only
2245        // teardown knows how many pending route.bind relays it aborted. It still
2246        // goes out before the GOODBYEs for the released routes, and its targets
2247        // were captured above, before teardown removed those routes.
2248        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2249            let abandoned = cleanup
2250                .as_ref()
2251                .map_or(0, |cleanup| cleanup.abandoned_relays);
2252            send_route_control_pushes(
2253                &self.forwarding,
2254                routes,
2255                ClientControlPush::RouteClosed {
2256                    module_id,
2257                    channels: Vec::new(),
2258                    reason,
2259                    drained,
2260                    abandoned,
2261                    excluded_subscriptions: 0,
2262                    terminal: Some(terminal),
2263                },
2264            );
2265        }
2266        if let Ok(cleanup) = cleanup {
2267            self.emit_route_goodbyes(cleanup.released);
2268        }
2269        // Signal the registration-release watch only now that BOTH registry and
2270        // forwarding teardown are done, so a supervisor waiting to spawn a
2271        // replacement never observes release while old routes still exist.
2272        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2273            crate::supervise::notify_registration_release();
2274            self.capability_evaluator.wake_deadline_loop();
2275            self.refresh_capability_requirements();
2276        }
2277        self.supervisor.remove_spawn_subscribers(connection_id);
2278        self.activity_streams
2279            .lock()
2280            .unwrap_or_else(|p| p.into_inner())
2281            .retain(|(id, _), task| {
2282                if *id == connection_id {
2283                    task.abort();
2284                    false
2285                } else {
2286                    true
2287                }
2288            });
2289        // Sync authority dies with its connection, so the owner's next
2290        // connection can take it; the owner's scopes stay as they are.
2291        self.hello_launch_nonces
2292            .lock()
2293            .unwrap_or_else(|poisoned| poisoned.into_inner())
2294            .forget(connection_id);
2295        self.scopes
2296            .write()
2297            .unwrap_or_else(|poisoned| poisoned.into_inner())
2298            .release_connection(connection_id);
2299        registrations
2300    }
2301
2302    pub(crate) fn handle_route_goodbye(
2303        &self,
2304        connection_id: ConnectionId,
2305        route_channel: u16,
2306        route_epoch: u32,
2307    ) -> Result<bool, RouterError> {
2308        debug!(
2309            connection_id = connection_id.get(),
2310            route_channel, route_epoch, "handling route GOODBYE"
2311        );
2312        let RouteRelease::Removed(released_route) = self
2313            .forwarding
2314            .release_client_route(connection_id, route_channel, route_epoch)
2315            .map_err(RouterError::Forwarding)?
2316        else {
2317            return Ok(false);
2318        };
2319        self.emit_route_goodbyes(vec![released_route]);
2320        Ok(true)
2321    }
2322
2323    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2324        for released in released_routes {
2325            let frame = match Frame::build_with_version(
2326                released.negotiated_ver,
2327                FrameType::Goodbye,
2328                control_flags(),
2329                released.channel,
2330                released.epoch,
2331                0,
2332                Vec::new(),
2333            ) {
2334                Ok(frame) => frame,
2335                Err(err) => {
2336                    warn!(
2337                        route_channel = released.channel,
2338                        error = %err,
2339                        "failed to build route GOODBYE frame"
2340                    );
2341                    continue;
2342                }
2343            };
2344            if !released.close_on_delivery_failure() {
2345                crate::forwarding::send_module_route_goodbye(
2346                    &self.counters,
2347                    &released.sink,
2348                    frame,
2349                    released.module_id.as_deref(),
2350                    "client route released",
2351                );
2352                continue;
2353            }
2354            if let Err(err) = released.sink.try_send(frame) {
2355                warn!(
2356                    target_connection_id = released.connection_id.get(),
2357                    route_channel = released.channel,
2358                    error = %err,
2359                    "route GOODBYE was not delivered to client; closing target connection"
2360                );
2361                if self
2362                    .forwarding
2363                    .escalate_client_delivery_failure(
2364                        released.connection_id,
2365                        released.channel,
2366                        released.epoch,
2367                        CloseReason::new(
2368                            "route_goodbye_delivery_failed",
2369                            format!(
2370                                "failed to enqueue route GOODBYE for channel {}: {err}",
2371                                released.channel
2372                            ),
2373                        ),
2374                        crate::forwarding::UndeliveredFrame {
2375                            module_id: released.module_id.as_deref(),
2376                            sink: &released.sink,
2377                        },
2378                    )
2379                    .unwrap_or(false)
2380                {
2381                    self.counters.increment_goodbye_relay_client_failed();
2382                }
2383            }
2384        }
2385    }
2386
2387    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2388    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2389    /// own commit failed after the module had already accepted). Without this, a
2390    /// module that accepts late keeps a binding subc has torn down, so a later
2391    /// frame on that module channel could misdeliver if the channel is reused.
2392    ///
2393    /// Never closes the shared module connection on failure: a dropped notification
2394    /// only wastes a bounded amount of warm module-side state, which the module's
2395    /// own idle reaper reclaims. Only call this once the route.bind relay was
2396    /// actually enqueued to the module — if the relay send itself failed, the
2397    /// module never created a binding and there is nothing to tear down.
2398    fn send_abandoned_route_bind_goodbye(
2399        &self,
2400        module_sink: &crate::FrameSink,
2401        negotiated_ver: u8,
2402        module_channel: u16,
2403        module_epoch: u32,
2404    ) {
2405        let frame = match Frame::build_with_version(
2406            negotiated_ver,
2407            FrameType::Goodbye,
2408            control_flags(),
2409            module_channel,
2410            module_epoch,
2411            0,
2412            Vec::new(),
2413        ) {
2414            Ok(frame) => frame,
2415            Err(err) => {
2416                warn!(
2417                    route_channel = module_channel,
2418                    error = %err,
2419                    "failed to build GOODBYE for abandoned route.bind"
2420                );
2421                return;
2422            }
2423        };
2424        crate::forwarding::send_module_route_goodbye(
2425            &self.counters,
2426            module_sink,
2427            frame,
2428            None,
2429            "abandoned route.bind",
2430        );
2431    }
2432
2433    fn handle_hello(
2434        &self,
2435        connection_id: ConnectionId,
2436        sink: Option<crate::FrameSink>,
2437        frame: Frame,
2438    ) -> Result<Vec<Frame>, RouterError> {
2439        debug!(
2440            connection_id = connection_id.get(),
2441            corr = frame.header.corr,
2442            "handling HELLO"
2443        );
2444        // A module connection has one identity for its entire lifetime. A second
2445        // registration would leave the old registry owner behind while replacing
2446        // its forwarding endpoint and launch nonce.
2447        if self
2448            .registry
2449            .get_module_by_connection(connection_id)
2450            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2451            .is_some()
2452        {
2453            return Ok(vec![control_error_frame(
2454                &frame,
2455                "invalid_hello",
2456                "connection is already registered as a module",
2457            )?]);
2458        }
2459        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2460            Ok(value) => value,
2461            Err(err) => {
2462                return Ok(vec![control_error_frame(
2463                    &frame,
2464                    "invalid_hello",
2465                    format!("malformed HELLO body: {err}"),
2466                )?])
2467            }
2468        };
2469        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2470            return Ok(vec![control_error_frame(
2471                &frame,
2472                "invalid_capability_grammar",
2473                err.to_string(),
2474            )?]);
2475        }
2476        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2477            return Ok(vec![control_error_frame(
2478                &frame,
2479                "invalid_manifest",
2480                err.to_string(),
2481            )?]);
2482        }
2483        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2484            return Ok(vec![control_error_frame(
2485                &frame,
2486                "invalid_event_declaration",
2487                err.to_string(),
2488            )?]);
2489        }
2490        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2491            return Ok(vec![control_error_frame(
2492                &frame,
2493                "invalid_manifest",
2494                err.to_string(),
2495            )?]);
2496        }
2497        if let Some(provenance) = hello_value
2498            .get("manifest")
2499            .and_then(|manifest| manifest.get("provenance"))
2500        {
2501            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2502                return Ok(vec![control_error_frame(
2503                    &frame,
2504                    "invalid_manifest",
2505                    format!("malformed manifest provenance: {err}"),
2506                )?]);
2507            }
2508        }
2509        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2510            let module_id = hello_value["manifest"]["module_id"]
2511                .as_str()
2512                .unwrap_or("<unknown>");
2513            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2514        }
2515        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2516            let module_id = hello_value["manifest"]["module_id"]
2517                .as_str()
2518                .unwrap_or("<unknown>");
2519            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2520        }
2521        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2522            let module_id = hello_value["manifest"]["module_id"]
2523                .as_str()
2524                .unwrap_or("<unknown>");
2525            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2526        }
2527        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2528            Ok(hello) => hello,
2529            Err(err) => {
2530                return Ok(vec![control_error_frame(
2531                    &frame,
2532                    "invalid_hello",
2533                    format!("malformed HELLO body: {err}"),
2534                )?])
2535            }
2536        };
2537
2538        if hello.protocol_ver != hello.manifest.protocol_ver {
2539            return Ok(vec![control_error_frame(
2540                &frame,
2541                "invalid_manifest",
2542                format!(
2543                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2544                    hello.protocol_ver, hello.manifest.protocol_ver
2545                ),
2546            )?]);
2547        }
2548
2549        if hello.manifest.module_id.trim().is_empty() {
2550            return Ok(vec![control_error_frame(
2551                &frame,
2552                "invalid_manifest",
2553                "manifest module_id must not be empty",
2554            )?]);
2555        }
2556
2557        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2558            Ok(negotiated_ver) => negotiated_ver,
2559            Err(message) => {
2560                return Ok(vec![control_error_frame(
2561                    &frame,
2562                    "version_unsupported",
2563                    message,
2564                )?])
2565            }
2566        };
2567
2568        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2569        // swap is open for this id, the only HELLO admitted as a second process
2570        // is the one carrying the candidate's launch nonce (the swap token), and
2571        // it registers into the candidate slot rather than being refused as a
2572        // duplicate. Run after the reserved gate, a reserved module's candidate
2573        // would be refused `reserved_module` for presenting a nonce that gate
2574        // does not know. See `SupervisorHandle::swap_hello_admission`.
2575        let swap_admission = self
2576            .supervisor
2577            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2578        if swap_admission == SwapHelloAdmission::Refused {
2579            warn!(
2580                module_id = %hello.manifest.module_id,
2581                connection_id = connection_id.get(),
2582                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2583            );
2584            return Ok(vec![control_error_frame(
2585                &frame,
2586                "swap_token_invalid",
2587                format!(
2588                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2589                    hello.manifest.module_id
2590                ),
2591            )?]);
2592        }
2593        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2594
2595        // A swap candidate proves its identity with the candidate nonce above.
2596        // Other registrations receive this catalog fact only when reserved-module
2597        // policy required and accepted a nonce; unreserved registrations require
2598        // no nonce check.
2599        let nonce_verified = if swap_candidate {
2600            true
2601        } else {
2602            match self.supervisor.reserved_hello_verification(
2603                &hello.manifest.module_id,
2604                hello.launch_nonce.as_deref(),
2605            ) {
2606                Ok(verified) => verified,
2607                Err(rejection) => {
2608                    let message = match rejection {
2609                        ReservedHelloRejection::Exact { module_id } => format!(
2610                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2611                        ),
2612                        ReservedHelloRejection::Prefix {
2613                            prefix,
2614                            owner_module_id,
2615                        } => format!(
2616                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2617                            hello.manifest.module_id
2618                        ),
2619                    };
2620                    return Ok(vec![control_error_frame(
2621                        &frame,
2622                        "reserved_module",
2623                        message,
2624                    )?]);
2625                }
2626            }
2627        };
2628
2629        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2630            &hello.manifest.module_id,
2631            hello.manifest.capabilities.as_ref(),
2632        );
2633        if let Some(refusal) = reserved_capability_refusals.first() {
2634            let capability = refusal.capability.clone();
2635            let bound_module = refusal.claimants[0].clone();
2636            log_duplicate_claim_events(reserved_capability_refusals);
2637            return Ok(vec![control_error_frame(
2638                &frame,
2639                "reserved_capability",
2640                format!(
2641                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2642                    capability, bound_module, hello.manifest.module_id
2643                ),
2644            )?]);
2645        }
2646
2647        // A connection that already opened client routes must not also register as
2648        // a module: cleanup would then release only one side and leak the other.
2649        if self
2650            .forwarding
2651            .connection_has_client_routes(connection_id)
2652            .map_err(RouterError::Forwarding)?
2653        {
2654            return Ok(vec![control_error_frame(
2655                &frame,
2656                "invalid_hello",
2657                "connection has open client routes and cannot also register as a module",
2658            )?]);
2659        }
2660
2661        // Kept for scope sync authority, which goes only to the connection that
2662        // presented the module's current launch nonce. Recorded before the
2663        // registration is attempted: a connection whose registration then fails
2664        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2665        self.hello_launch_nonces
2666            .lock()
2667            .unwrap_or_else(|poisoned| poisoned.into_inner())
2668            .record(connection_id, hello.launch_nonce.as_deref());
2669        let control_ops = effective_module_control_ops(hello.control_ops);
2670        // Built before anything is registered so an encoding failure leaves no
2671        // registry or forwarding state behind.
2672        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2673        if swap_candidate {
2674            return self.register_swap_candidate(
2675                connection_id,
2676                sink,
2677                &frame,
2678                hello.manifest,
2679                negotiated_ver,
2680                control_ops,
2681                hello_ack,
2682            );
2683        }
2684        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2685        // is refused, but `catalog.list` does not show the module and the
2686        // catalog generation does not move until it is published below, once
2687        // its forwarding endpoint exists. Listing it now would advertise a
2688        // module that `route.open` still refuses for having no forwarding
2689        // connection.
2690        let registration = match self
2691            .registry
2692            .register_unpublished_verified_with_control_ops(
2693                hello.manifest,
2694                negotiated_ver,
2695                connection_id,
2696                control_ops,
2697                nonce_verified,
2698            ) {
2699            Ok(registration) => registration,
2700            Err(RegistryError::DuplicateModuleId { module_id }) => {
2701                return Ok(vec![control_error_frame(
2702                    &frame,
2703                    "duplicate_module_id",
2704                    format!(
2705                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2706                    ),
2707                )?])
2708            }
2709            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2710                return Ok(vec![control_error_frame(
2711                    &frame,
2712                    "invalid_module_id",
2713                    err.to_string(),
2714                )?])
2715            }
2716            Err(err) => {
2717                return Ok(vec![control_error_frame(
2718                    &frame,
2719                    "registry_error",
2720                    err.to_string(),
2721                )?])
2722            }
2723        };
2724        #[cfg(test)]
2725        self.hold_between_registry_and_forwarding();
2726
2727        let reply = if let Some(sink) = sink {
2728            // The forwarding table's module store is also the daemon-to-module
2729            // control-RPC lane, so every HELLO gets a live endpoint even when the
2730            // manifest has no routable provider role. Non-routable modules still
2731            // cannot receive route.bind in production: `handle_route_open` checks
2732            // the registry manifest with `target_has_required_role` before the
2733            // only production call to `begin_route_bind_relay_for` below that
2734            // route.open path. The remaining direct relay callers are unit tests
2735            // and benchmark harnesses that construct forwarding state explicitly.
2736            //
2737            // The HELLO_ACK is queued by the forwarding table itself, before the
2738            // endpoint becomes visible, and is NOT returned as a reply. A module
2739            // reads HELLO_ACK first and exits on anything else; a reply is only
2740            // written after this handler returns, by which time a route.open on
2741            // another connection could already have queued a route.bind request
2742            // for this module ahead of it.
2743            let concurrency = manifest_concurrency(&registration.manifest);
2744            if let Err(err) = self.forwarding.register_module_connection_acked(
2745                connection_id,
2746                registration.manifest.module_id.clone(),
2747                negotiated_ver,
2748                concurrency,
2749                sink,
2750                hello_ack,
2751            ) {
2752                // Forwarding registration failed, so there is no forwarding
2753                // state to tear down. Remove the registry entry, which was never
2754                // published and so never listed or counted in the catalog
2755                // generation, and signal the release watch directly.
2756                if matches!(
2757                    self.deregister_connection(
2758                        connection_id,
2759                        RegistrationEndReason::RegistrationFailed,
2760                    ),
2761                    Ok(r) if !r.is_empty()
2762                ) {
2763                    crate::supervise::notify_registration_release();
2764                }
2765                return Ok(vec![control_error_frame(
2766                    &frame,
2767                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2768                        "invalid_hello"
2769                    } else {
2770                        forwarding_error_code(&err)
2771                    },
2772                    err.to_string(),
2773                )?]);
2774            }
2775            Vec::new()
2776        } else {
2777            // No sink means no forwarding endpoint, so nothing can be routed
2778            // ahead of the ack; it goes out as the reply.
2779            vec![hello_ack]
2780        };
2781
2782        // The module is routable now (or, without a sink, as routable as it
2783        // will ever be), so the catalog may list it. Publishing bumps the
2784        // catalog generation in the same registry critical section that makes
2785        // the module listed.
2786        if !self
2787            .registry
2788            .publish_registration(connection_id)
2789            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2790        {
2791            debug!(
2792                module_id = %registration.manifest.module_id,
2793                connection_id = connection_id.get(),
2794                "registration was removed before it could be published"
2795            );
2796        }
2797
2798        // Exposure over assumption: Concurrency's serde default is pinned to the
2799        // pre-field behavior (ModuleManaged), so a management surface that is
2800        // genuinely Serial and just never declared it inherits concurrent
2801        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2802        // turns "no module has been bitten yet" into the checkable claim "no
2803        // module is exposed" -- one read of the boot log instead of a fleet
2804        // audit. Detected from the raw HELLO bytes because the serde default
2805        // deliberately erases the absent/declared distinction from the type.
2806        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2807            info!(
2808                module_id = %registration.manifest.module_id,
2809                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2810            );
2811        }
2812
2813        self.apply_registration_capabilities(&registration);
2814
2815        self.supervisor
2816            .record_activity_registration(&registration.manifest.module_id);
2817
2818        info!(
2819            module_id = %registration.manifest.module_id,
2820            module_version = %registration.manifest.module_version,
2821            negotiated_ver,
2822            routable_provider = manifest_provides_routable_role(&registration.manifest),
2823            connection_id = connection_id.get(),
2824            "module registered"
2825        );
2826
2827        Ok(reply)
2828    }
2829
2830    /// Register a HELLO the swap gate admitted into the candidate slot of the
2831    /// registry and of forwarding, where it is reachable over its own
2832    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2833    /// nothing routes to it until the supervisor cuts over.
2834    ///
2835    /// Registry first (unpublished), then forwarding, then publication, the
2836    /// same order as an ordinary HELLO; a forwarding failure removes the
2837    /// registry entry again. The swap supervisor treats a published, ready
2838    /// candidate as registered and promotes it, and promotion needs the
2839    /// candidate's forwarding endpoint, so the candidate is published only
2840    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2841    /// recompute capability verdicts or run the capability deny census
2842    /// (`enforce_capability_denies`): both are about the module in the active
2843    /// slot, which `route.open` can target. A candidate sits in the candidate
2844    /// slot, so it neither provides capabilities to other modules nor can be
2845    /// routed to until promotion moves it into the active slot.
2846    #[allow(clippy::too_many_arguments)]
2847    fn register_swap_candidate(
2848        &self,
2849        connection_id: ConnectionId,
2850        sink: Option<crate::FrameSink>,
2851        frame: &Frame,
2852        manifest: ModuleManifest,
2853        negotiated_ver: u8,
2854        control_ops: Vec<String>,
2855        hello_ack: Frame,
2856    ) -> Result<Vec<Frame>, RouterError> {
2857        let module_id = manifest.module_id.clone();
2858        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2859            manifest,
2860            negotiated_ver,
2861            connection_id,
2862            control_ops,
2863        ) {
2864            Ok(registration) => registration,
2865            Err(RegistryError::DuplicateModuleId { module_id }) => {
2866                return Ok(vec![control_error_frame(
2867                    frame,
2868                    "duplicate_module_id",
2869                    format!(
2870                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2871                    ),
2872                )?])
2873            }
2874            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2875                return Ok(vec![control_error_frame(
2876                    frame,
2877                    "invalid_module_id",
2878                    err.to_string(),
2879                )?])
2880            }
2881            Err(err) => {
2882                return Ok(vec![control_error_frame(
2883                    frame,
2884                    "registry_error",
2885                    err.to_string(),
2886                )?])
2887            }
2888        };
2889        #[cfg(test)]
2890        self.hold_between_registry_and_forwarding();
2891        let reply = if let Some(sink) = sink {
2892            // Same ordering as an ordinary HELLO: the forwarding table queues
2893            // the HELLO_ACK before the candidate endpoint is inserted, because
2894            // a module exits if its first frame after HELLO is anything else.
2895            let concurrency = manifest_concurrency(&registration.manifest);
2896            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2897                connection_id,
2898                module_id.clone(),
2899                negotiated_ver,
2900                concurrency,
2901                sink,
2902                hello_ack,
2903            ) {
2904                if matches!(
2905                    self.deregister_connection(
2906                        connection_id,
2907                        RegistrationEndReason::RegistrationFailed,
2908                    ),
2909                    Ok(r) if !r.is_empty()
2910                ) {
2911                    crate::supervise::notify_registration_release();
2912                }
2913                return Ok(vec![control_error_frame(
2914                    frame,
2915                    forwarding_error_code(&err),
2916                    err.to_string(),
2917                )?]);
2918            }
2919            Vec::new()
2920        } else {
2921            vec![hello_ack]
2922        };
2923        if !self
2924            .registry
2925            .publish_registration(connection_id)
2926            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2927        {
2928            debug!(
2929                module_id = %module_id,
2930                connection_id = connection_id.get(),
2931                "swap candidate registration was removed before it could be published"
2932            );
2933        }
2934        self.supervisor.mark_swap_candidate_admitted(&module_id);
2935        info!(
2936            module_id = %module_id,
2937            module_version = %registration.manifest.module_version,
2938            negotiated_ver,
2939            ready = registration.ready,
2940            connection_id = connection_id.get(),
2941            "swap candidate registered; not routable until cutover"
2942        );
2943        Ok(reply)
2944    }
2945
2946    fn build_hello_ack(
2947        &self,
2948        frame: &Frame,
2949        negotiated_ver: u8,
2950        module_id: &str,
2951    ) -> Result<Frame, RouterError> {
2952        let ack = ModuleHelloAckBody {
2953            negotiated_ver,
2954            subc_ops: module_subc_ops(),
2955            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2956            storage: self
2957                .storage_config
2958                .as_ref()
2959                .map(|cfg| cfg.descriptor_for(module_id)),
2960            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2961        };
2962        let body = serde_json::to_vec(&ack).map_err(|err| {
2963            RouterError::backend(
2964                0,
2965                frame.header.corr,
2966                format!("failed to encode HELLO_ACK: {err}"),
2967            )
2968        })?;
2969
2970        Frame::build_with_version(
2971            negotiated_ver,
2972            FrameType::HelloAck,
2973            control_flags(),
2974            0,
2975            0,
2976            frame.header.corr,
2977            body,
2978        )
2979        .map_err(RouterError::FrameBuild)
2980    }
2981
2982    async fn handle_client_control_request(
2983        &self,
2984        ctx: &RouteCtx,
2985        frame: Frame,
2986        request: ClientControlRequest,
2987        route_open_timings: &mut Option<RouteOpenTimings>,
2988    ) -> Result<Vec<Frame>, RouterError> {
2989        if let Some(operation) = privileged_operation(&request) {
2990            let caller = self.resolve_caller(ctx.connection_id).await;
2991            let op = client_control_request_op(&request);
2992            let text = prompt_text(operation, &caller);
2993            match self.privileged_operator_prompt {
2994                PrivilegedOperatorPrompt::Log => {
2995                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2996                        target = ?operation, caller = %caller.description(), summary = %text,
2997                        "privileged request would prompt the operator");
2998                }
2999                PrivilegedOperatorPrompt::Enforce => {
3000                    let key =
3001                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3002                    let outcome = self
3003                        .forwarding
3004                        .operator_confirms()
3005                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
3006                        .await;
3007                    if outcome.code != "confirmed" {
3008                        return Ok(vec![outcome.frame(&frame)]);
3009                    }
3010                }
3011            }
3012        }
3013        match request {
3014            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3015            ClientControlRequest::CatalogList { module_id } => {
3016                self.handle_catalog_list(frame, module_id)
3017            }
3018            ClientControlRequest::RouteOpen {
3019                target,
3020                identity,
3021                consumer_identity,
3022                consumer_capabilities,
3023                role_versions,
3024                admission_facts,
3025                scope,
3026            } => {
3027                self.handle_route_open(
3028                    ctx,
3029                    frame,
3030                    RouteOpenRequest {
3031                        target,
3032                        identity,
3033                        consumer_identity,
3034                        consumer_capabilities,
3035                        role_versions,
3036                        admission_facts,
3037                        scope,
3038                    },
3039                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3040                )
3041                .await
3042            }
3043            ClientControlRequest::RoutePoll {
3044                route_channel,
3045                route_epoch,
3046                kind,
3047            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3048            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
3049            ClientControlRequest::SupervisorSpawnSnapshot {} => {
3050                self.handle_supervisor_spawn_snapshot(frame)
3051            }
3052            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3053                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3054            }
3055            ClientControlRequest::SupervisorRestart {
3056                module_id,
3057                drain_timeout_ms,
3058            } => {
3059                self.log_supervisor_request_received(
3060                    ctx,
3061                    frame.header.corr,
3062                    ops::SUPERVISOR_RESTART,
3063                    Some(&module_id),
3064                    None,
3065                )?;
3066                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3067                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3068                    .await
3069            }
3070            ClientControlRequest::SupervisorSwap {
3071                module_id,
3072                ready_timeout_ms,
3073            } => {
3074                self.log_supervisor_request_received(
3075                    ctx,
3076                    frame.header.corr,
3077                    ops::SUPERVISOR_SWAP,
3078                    Some(&module_id),
3079                    None,
3080                )?;
3081                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3082                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3083                    .await
3084            }
3085            ClientControlRequest::SupervisorReload { module_id } => {
3086                self.log_supervisor_request_received(
3087                    ctx,
3088                    frame.header.corr,
3089                    ops::SUPERVISOR_RELOAD,
3090                    Some(&module_id),
3091                    None,
3092                )?;
3093                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3094                self.handle_supervisor_reload(frame, module_id, requester)
3095                    .await
3096            }
3097            ClientControlRequest::SupervisorRescan { preview } => {
3098                if !preview {
3099                    self.log_supervisor_request_received(
3100                        ctx,
3101                        frame.header.corr,
3102                        ops::SUPERVISOR_RESCAN,
3103                        None,
3104                        None,
3105                    )?;
3106                }
3107                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3108                self.handle_supervisor_rescan(
3109                    frame,
3110                    if preview {
3111                        RescanMode::Preview
3112                    } else {
3113                        RescanMode::Execute
3114                    },
3115                    requester,
3116                )
3117                .await
3118            }
3119            ClientControlRequest::SupervisorRescanChecked {
3120                preview,
3121                candidate,
3122                expect,
3123            } => {
3124                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
3125                    .await
3126            }
3127            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3128                self.log_supervisor_request_received(
3129                    ctx,
3130                    frame.header.corr,
3131                    ops::SUPERVISOR_RELEASE_RESERVED,
3132                    Some(&module_id),
3133                    None,
3134                )?;
3135                self.handle_supervisor_release_reserved(frame, module_id)
3136                    .await
3137            }
3138            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3139                self.log_supervisor_request_received(
3140                    ctx,
3141                    frame.header.corr,
3142                    ops::SUPERVISOR_SET_ENABLED,
3143                    Some(&module_id),
3144                    Some(enabled),
3145                )?;
3146                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3147                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3148                    .await
3149            }
3150            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3151                self.handle_supervisor_health_probe(frame, module_id).await
3152            }
3153            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3154            ClientControlRequest::SupervisorRoutes { module_id } => {
3155                self.handle_supervisor_routes(frame, module_id)
3156            }
3157            ClientControlRequest::SupervisorProvenance { module_id } => {
3158                self.handle_supervisor_provenance(frame, module_id).await
3159            }
3160            ClientControlRequest::SupervisorStderrTail {
3161                module_id,
3162                max_lines,
3163                max_bytes,
3164            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3165            ClientControlRequest::SupervisorTerminals { module_id } => {
3166                self.handle_supervisor_terminals(frame, module_id).await
3167            }
3168        }
3169    }
3170
3171    fn log_supervisor_request_received(
3172        &self,
3173        ctx: &RouteCtx,
3174        corr: u64,
3175        op: &'static str,
3176        module_id: Option<&str>,
3177        enabled: Option<bool>,
3178    ) -> Result<(), RouterError> {
3179        let caller = self.supervisor_requester(ctx, corr)?;
3180        let caller = principal_label(&caller);
3181        // `Option` fields are recorded only when present, so a request with no
3182        // module id (rescan) or no enabled flag simply omits that field.
3183        info!(
3184            target: "control",
3185            op,
3186            module_id,
3187            enabled,
3188            connection_id = ctx.connection_id.get(),
3189            caller = %caller,
3190            "supervisor request received"
3191        );
3192        Ok(())
3193    }
3194
3195    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3196        Ok(self
3197            .registry
3198            .get_module_by_connection(ctx.connection_id)
3199            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3200            .map(|registration| Principal::Reserved {
3201                module_id: registration.manifest.module_id,
3202            })
3203            .unwrap_or(Principal::Direct))
3204    }
3205
3206    fn handle_module_control_request(
3207        &self,
3208        connection_id: ConnectionId,
3209        frame: Frame,
3210        request: ModuleControlRequestFromModule,
3211    ) -> Result<Vec<Frame>, RouterError> {
3212        match request {
3213            ModuleControlRequestFromModule::CatalogUpdate {
3214                provides,
3215                capabilities,
3216                ready,
3217            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3218            ModuleControlRequestFromModule::LiveRoots {} => {
3219                let registered = self
3220                    .registry
3221                    .get_module_by_connection(connection_id)
3222                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3223                let Some(registration) = registered else {
3224                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3225                };
3226                let response = self
3227                    .forwarding
3228                    .live_roots(&registration.manifest.module_id)
3229                    .map_err(RouterError::Forwarding)?;
3230                Ok(vec![control_response_body_frame(
3231                    &frame,
3232                    &response,
3233                    "ModuleControlResponseToModule::LiveRoots",
3234                )?])
3235            }
3236            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3237                self.handle_scope_sync(connection_id, frame, generation, scopes)
3238            }
3239            ModuleControlRequestFromModule::ScopeApply {
3240                generation,
3241                upsert,
3242                end,
3243            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3244            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3245                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3246            }
3247        }
3248    }
3249
3250    fn handle_scope_served_by(
3251        &self,
3252        connection_id: ConnectionId,
3253        frame: Frame,
3254    ) -> Result<Vec<Frame>, RouterError> {
3255        let registered = self
3256            .registry
3257            .get_module_by_connection(connection_id)
3258            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3259        let Some(registered) = registered else {
3260            return Ok(vec![control_error_frame(
3261                &frame,
3262                "not_registered",
3263                "scope.served_by requires an active module registration owned by this connection",
3264            )?]);
3265        };
3266        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3267        {
3268            Ok(request) => request,
3269            Err(err) => {
3270                return Ok(vec![control_error_frame(
3271                    &frame,
3272                    "invalid_control_body",
3273                    format!("malformed scope.served_by body: {err}"),
3274                )?])
3275            }
3276        };
3277        let answer = self
3278            .scopes
3279            .read()
3280            .unwrap_or_else(|poisoned| poisoned.into_inner())
3281            .served_by(
3282                &registered.manifest.module_id,
3283                &request.scope_ref,
3284                request.scope_epoch,
3285                &request.module_id,
3286            );
3287        let reply = subc_control::ScopeServedByReply::new(
3288            answer,
3289            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3290        );
3291        Ok(vec![control_response_body_frame(
3292            &frame,
3293            &reply,
3294            "ScopeServedByReply",
3295        )?])
3296    }
3297
3298    fn handle_operator_confirm(
3299        &self,
3300        ctx: &RouteCtx,
3301        frame: Frame,
3302    ) -> Result<Vec<Frame>, RouterError> {
3303        use crate::operator_confirm::{audit, Outcome};
3304        let registration = self
3305            .registry
3306            .get_module_by_connection(ctx.connection_id)
3307            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3308        let Some(registration) = registration else {
3309            let outcome = Outcome::refusal("not_registered");
3310            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3311            return Ok(vec![outcome.frame(&frame)]);
3312        };
3313        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3314            Ok(request) => request,
3315            Err(_) => {
3316                let outcome = Outcome::refusal("invalid_control_body");
3317                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3318                return Ok(vec![outcome.frame(&frame)]);
3319            }
3320        };
3321        let module_id = registration.manifest.module_id;
3322        // Read the launch nonce (under its own lock) before taking the forwarding
3323        // table's lock below: holding forwarding while waiting on another daemon
3324        // lock risks a lock-order deadlock with paths that take them the other way.
3325        let nonce = self
3326            .hello_launch_nonces
3327            .lock()
3328            .unwrap_or_else(|p| p.into_inner())
3329            .nonce(ctx.connection_id)
3330            .map(str::to_owned);
3331        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3332            self.supervisor
3333                .spawned_consumer_authorized(&module_id, nonce)
3334        });
3335        let confirms = self.forwarding.operator_confirms();
3336        self.forwarding
3337            .with_operator_route(
3338                ctx.connection_id,
3339                request.route_channel,
3340                request.route_epoch,
3341                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3342            )
3343            .map_err(RouterError::Forwarding)
3344    }
3345
3346    /// Resolve folders before taking the scope-table lock: a slow filesystem
3347    /// must not hold up admissions for unrelated scoped routes.
3348    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3349        let mut scopes = ScopeRecords::from(scopes);
3350        for (index, record) in scopes.records.iter_mut().enumerate() {
3351            if let Some(refusal) = Self::canonicalize_workdir(record) {
3352                scopes.refusals.insert(index, refusal);
3353            }
3354        }
3355        scopes
3356    }
3357
3358    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3359        let workdir = record.workdir.as_mut()?;
3360        let path = std::path::Path::new(workdir);
3361        if !path.is_absolute() {
3362            return Some((
3363                error_codes::INVALID_WORKDIR,
3364                "workdir must be absolute".to_string(),
3365            ));
3366        }
3367        match ProjectRootId::from_path_allowing_missing(path) {
3368            Ok(root) => match root.as_path().to_str() {
3369                Some(canonical) => {
3370                    *workdir = canonical.to_string();
3371                    None
3372                }
3373                None => Some((
3374                    error_codes::INVALID_WORKDIR,
3375                    "canonical workdir is not UTF-8".to_string(),
3376                )),
3377            },
3378            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3379        }
3380    }
3381
3382    /// `scope.sync`: the owner is the module registered on this connection.
3383    /// A connection with no registration (every client connection, `direct`
3384    /// included) is refused `not_registered` before the table is consulted.
3385    fn handle_scope_sync(
3386        &self,
3387        connection_id: ConnectionId,
3388        frame: Frame,
3389        generation: u64,
3390        scopes: Vec<ScopeRecord>,
3391    ) -> Result<Vec<Frame>, RouterError> {
3392        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3393    }
3394
3395    fn handle_scope_change(
3396        &self,
3397        connection_id: ConnectionId,
3398        frame: Frame,
3399        generation: u64,
3400        scopes: Vec<ScopeRecord>,
3401        end: Option<Vec<ScopeEnd>>,
3402    ) -> Result<Vec<Frame>, RouterError> {
3403        let is_apply = end.is_some();
3404        let Some(registration) = self
3405            .registry
3406            .get_module_by_connection(connection_id)
3407            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3408        else {
3409            return Ok(vec![control_error_frame(
3410                &frame,
3411                "not_registered",
3412                "scope.sync and scope.apply require an active module registration owned by this connection",
3413            )?]);
3414        };
3415        let owner = registration.manifest.module_id;
3416        let scopes = Self::prepare_scope_records(scopes);
3417        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3418        let is_current_launch = |connection: ConnectionId| {
3419            self.hello_launch_nonces
3420                .lock()
3421                .unwrap_or_else(|poisoned| poisoned.into_inner())
3422                .presented(connection, current_nonce.as_deref())
3423        };
3424        // Lock order is the scope table, then the forwarding table: the new
3425        // tags are published, and the routes the change closes are selected,
3426        // while the scope table is still write-locked, so no admission can read
3427        // a record whose tag is not yet published.
3428        let mut table = self
3429            .scopes
3430            .write()
3431            .unwrap_or_else(|poisoned| poisoned.into_inner());
3432        let outcome = match end {
3433            Some(end) => table.apply(
3434                &owner,
3435                connection_id,
3436                is_current_launch,
3437                generation,
3438                scopes,
3439                end,
3440            ),
3441            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3442        };
3443        let drained = match &outcome {
3444            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3445            Err(_) => Vec::new(),
3446        };
3447        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3448        drop(table);
3449        if has_deadlines {
3450            self.scope_deadline_added.notify_one();
3451        }
3452        match outcome {
3453            Ok(applied) => {
3454                let counts = ScopeOutcomeCounts::of(&applied.results);
3455                info!(
3456                    owner = %owner,
3457                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3458                    generation,
3459                    records = applied.results.len(),
3460                    created = counts.created,
3461                    replaced = counts.replaced,
3462                    updated = counts.updated,
3463                    unchanged = counts.unchanged,
3464                    refused = counts.refused,
3465                    ended = applied.ended.len(),
3466                    tag_changes = applied.tag_changes.len(),
3467                    routes_closed = drained.len(),
3468                    "scope change accepted"
3469                );
3470                // An accepted scope change can still refuse individual records, and the
3471                // owner is the only party that sees the reply. Name them here so
3472                // an operator can tell a refused session from a missing one
3473                // without the owner's logs. Capped so a call that refuses
3474                // thousands cannot flood the log; the count above is complete.
3475                for refused in applied
3476                    .results
3477                    .iter()
3478                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3479                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3480                {
3481                    warn!(
3482                        owner = %owner,
3483                        generation,
3484                        scope_ref = %refused.scope_ref,
3485                        scope_epoch = refused.scope_epoch,
3486                        code = refused.code.as_deref().unwrap_or(""),
3487                        "scope record refused"
3488                    );
3489                }
3490                self.close_scope_drained_routes(drained);
3491                let response = if is_apply {
3492                    ModuleControlResponseToModule::ScopeApply {
3493                        generation,
3494                        results: applied.results,
3495                        end_results: applied.end_results,
3496                        ended: applied.ended,
3497                    }
3498                } else {
3499                    ModuleControlResponseToModule::ScopeSync {
3500                        generation,
3501                        results: applied.results,
3502                        ended: applied.ended,
3503                    }
3504                };
3505                Ok(vec![control_response_body_frame(
3506                    &frame,
3507                    &response,
3508                    "scope change response",
3509                )?])
3510            }
3511            Err(refusal) => {
3512                info!(
3513                    owner = %owner,
3514                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3515                    generation,
3516                    code = refusal.code,
3517                    "scope change refused"
3518                );
3519                Ok(vec![control_error_frame(
3520                    &frame,
3521                    refusal.code,
3522                    refusal.message,
3523                )?])
3524            }
3525        }
3526    }
3527
3528    /// Tell both ends of each route a scope change closed. The module gets a
3529    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3530    /// the client's route handle. The client also gets `route.closed` with the
3531    /// scope reason, one push per module and reason, so it can tell a revoked
3532    /// route from an ordinary close and not reopen it.
3533    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3534        if drained.is_empty() {
3535            return;
3536        }
3537        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3538            BTreeMap::new();
3539        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3540        for route in drained {
3541            warn!(
3542                module_id = %route.module_id,
3543                reason = ?route.reason,
3544                client_connection_id = route.client.connection_id.get(),
3545                route_channel = route.client.channel,
3546                "closing route because its scope changed"
3547            );
3548            pushes
3549                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3550                .or_insert_with(|| (route.reason, Vec::new()))
3551                .1
3552                .push(EndpointRoute {
3553                    goodbye_target: route.client.clone(),
3554                    principal: Principal::Unverified,
3555                    bound_at: Instant::now(),
3556                    draining: false,
3557                    drain_reason: None,
3558                });
3559            goodbyes.push(route.module);
3560            goodbyes.push(route.client);
3561        }
3562        for ((module_id, _), (reason, routes)) in pushes {
3563            send_route_control_pushes(
3564                &self.forwarding,
3565                routes,
3566                ClientControlPush::RouteClosed {
3567                    module_id,
3568                    channels: Vec::new(),
3569                    reason,
3570                    drained: false,
3571                    abandoned: 0,
3572                    excluded_subscriptions: 0,
3573                    terminal: Some(false),
3574                },
3575            );
3576        }
3577        self.emit_route_goodbyes(goodbyes);
3578    }
3579
3580    /// `scope.describe`: any registered module may read any scope, because a
3581    /// provider must read the scope a route it serves is stamped with.
3582    fn handle_scope_describe(
3583        &self,
3584        connection_id: ConnectionId,
3585        frame: Frame,
3586        owner: Principal,
3587        scope_ref: String,
3588    ) -> Result<Vec<Frame>, RouterError> {
3589        let registered = self
3590            .registry
3591            .get_module_by_connection(connection_id)
3592            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3593        let Some(registered) = registered else {
3594            return Ok(vec![control_error_frame(
3595                &frame,
3596                "not_registered",
3597                "scope.describe requires an active module registration owned by this connection",
3598            )?]);
3599        };
3600        let mut description = self
3601            .scopes
3602            .read()
3603            .unwrap_or_else(|poisoned| poisoned.into_inner())
3604            .describe(&owner, &scope_ref);
3605        // Describe readers also decode strict ScopeAttributes. Attribution is
3606        // safe to omit unless the requesting module declares it understands it.
3607        if !registered
3608            .manifest
3609            .capabilities
3610            .as_ref()
3611            .is_some_and(|caps| {
3612                caps.provides
3613                    .iter()
3614                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3615            })
3616        {
3617            if let Some(stamp) = description.stamp.as_mut() {
3618                stamp.attributes.on_behalf_of_agent = None;
3619            }
3620        }
3621        if !registered
3622            .manifest
3623            .capabilities
3624            .as_ref()
3625            .is_some_and(|caps| {
3626                caps.provides
3627                    .iter()
3628                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3629            })
3630        {
3631            if let Some(stamp) = description.stamp.as_mut() {
3632                stamp.attributes.flow_version = None;
3633                stamp.attributes.flow_code_hash = None;
3634                stamp.attributes.flow_install_id = None;
3635            }
3636        }
3637        let owner_configured = match &owner {
3638            // Ask whether the owner is configured (`is_configured`), not
3639            // whether it is on the roster (`get(..).is_some()`): a supervised
3640            // module's process can register and describe a scope before the
3641            // supervisor has put it on the roster.
3642            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3643            _ => false,
3644        };
3645        let response = ModuleControlResponseToModule::ScopeDescribe {
3646            status: description.status,
3647            scope_epoch: description.scope_epoch,
3648            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3649            owner_synced: description.owner_synced,
3650            owner_configured,
3651            scope: description.stamp,
3652        };
3653        Ok(vec![control_response_body_frame(
3654            &frame,
3655            &response,
3656            "ModuleControlResponseToModule::ScopeDescribe",
3657        )?])
3658    }
3659
3660    fn handle_catalog_update(
3661        &self,
3662        connection_id: ConnectionId,
3663        frame: Frame,
3664        provides: Vec<ProviderRole>,
3665        capabilities: Option<CapabilityDeclarations>,
3666        ready: Option<bool>,
3667    ) -> Result<Vec<Frame>, RouterError> {
3668        self.refresh_capability_requirements();
3669        let Some(registration) = self
3670            .registry
3671            .get_module_by_connection(connection_id)
3672            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3673        else {
3674            return Ok(vec![control_error_frame(
3675                &frame,
3676                "not_registered",
3677                "catalog.update requires an active module registration owned by this connection",
3678            )?]);
3679        };
3680
3681        if let Some(message) =
3682            catalog_update_frozen_field_message(&registration.manifest, &provides)
3683        {
3684            return Ok(vec![control_error_frame(
3685                &frame,
3686                "catalog_update_frozen_field",
3687                message,
3688            )?]);
3689        }
3690
3691        let mut candidate = registration.manifest.clone();
3692        candidate.provides = provides.clone();
3693        candidate.capabilities = capabilities
3694            .clone()
3695            .or_else(|| registration.manifest.capabilities.clone());
3696        if let Some(rich_ui) = &candidate.rich_ui {
3697            if let Err(err) = rich_ui.validate() {
3698                return Ok(vec![control_error_frame(
3699                    &frame,
3700                    "invalid_manifest",
3701                    err.to_string(),
3702                )?]);
3703            }
3704        }
3705        if let Err(err) = candidate.validate_capability_grammar() {
3706            return Ok(vec![control_error_frame(
3707                &frame,
3708                "invalid_capability_grammar",
3709                err.to_string(),
3710            )?]);
3711        }
3712
3713        // Updates must honor the same reserved owner as initial registration;
3714        // otherwise an empty HELLO could acquire the claim after admission.
3715        let mut conflicts = self
3716            .capability_evaluator
3717            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3718        if let Some(conflict) = conflicts.first() {
3719            let message = format!(
3720                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3721                conflict.capability, conflict.claimants[0], candidate.module_id
3722            );
3723            for conflict in &mut conflicts {
3724                conflict.source = DuplicateClaimSource::CatalogUpdate;
3725            }
3726            log_duplicate_claim_events(conflicts);
3727            return Ok(vec![control_error_frame(
3728                &frame,
3729                "reserved_capability",
3730                message,
3731            )?]);
3732        }
3733
3734        let updated = self
3735            .registry
3736            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3737            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3738        if updated.is_none() {
3739            return Ok(vec![control_error_frame(
3740                &frame,
3741                "not_registered",
3742                "catalog.update requires an active module registration owned by this connection",
3743            )?]);
3744        }
3745        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3746            log_duplicate_claim_events(
3747                self.capability_evaluator
3748                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3749            );
3750        }
3751        if capability_census_trigger(
3752            registration.manifest.capabilities.as_ref(),
3753            updated
3754                .as_ref()
3755                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3756        ) {
3757            self.enforce_capability_denies();
3758        }
3759        self.refresh_capability_requirements();
3760
3761        let response = ModuleControlResponseToModule::CatalogUpdate {};
3762        control_response_body_frame(
3763            &frame,
3764            &response,
3765            "ModuleControlResponseToModule::CatalogUpdate",
3766        )
3767        .map(|frame| vec![frame])
3768    }
3769
3770    pub(crate) fn with_activity_ledger_config(
3771        mut self,
3772        run_dir: &Path,
3773        incarnation: String,
3774        config: crate::activity::ActivityConfig,
3775    ) -> Self {
3776        self.activity = Some(crate::activity::ActivityLedger::open_config(
3777            run_dir.join("activity.jsonl"),
3778            incarnation,
3779            config,
3780        ));
3781        self.supervisor.set_activity(self.activity.clone());
3782        self
3783    }
3784
3785    pub(crate) async fn configure_activity(
3786        &self,
3787        config: crate::activity::ActivityConfig,
3788    ) -> Result<(), &'static str> {
3789        match &self.activity {
3790            Some(ledger) => ledger.configure(config).await,
3791            None => Ok(()),
3792        }
3793    }
3794
3795    fn cancel_activity_subscription(
3796        &self,
3797        ctx: &RouteCtx,
3798        frame: &Frame,
3799    ) -> Result<bool, RouterError> {
3800        let task = self
3801            .activity_streams
3802            .lock()
3803            .unwrap_or_else(|p| p.into_inner())
3804            .remove(&(ctx.connection_id, frame.header.corr));
3805        let Some(task) = task else {
3806            return Ok(false);
3807        };
3808        task.abort();
3809        let end = Frame::build_with_version(
3810            response_version(frame),
3811            FrameType::StreamEnd,
3812            control_flags(),
3813            0,
3814            0,
3815            frame.header.corr,
3816            Vec::new(),
3817        )
3818        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3819        let sink = ctx.egress.clone();
3820        tokio::spawn(async move {
3821            let _ = sink.send(end).await;
3822        });
3823        Ok(true)
3824    }
3825
3826    async fn handle_activity_read(
3827        &self,
3828        ctx: &RouteCtx,
3829        frame: Frame,
3830    ) -> Result<Vec<Frame>, RouterError> {
3831        use subc_control::{
3832            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3833            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3834        };
3835        #[derive(Deserialize)]
3836        struct Identity {
3837            #[serde(default)]
3838            consumer_identity: Option<ConsumerIdentity>,
3839        }
3840        // A registered module connection already identifies its caller. Ignore
3841        // consumer_identity there so a module cannot borrow a mirror's identity.
3842        // Only separate consumer connections use the Direct/nonce trust boundary.
3843        let principal = match self
3844            .forwarding
3845            .module_id_for_connection(ctx.connection_id)
3846            .map_err(RouterError::Forwarding)?
3847        {
3848            Some(module_id) => Ok(Principal::Reserved { module_id }),
3849            None => {
3850                let identity: Identity = match serde_json::from_slice(&frame.body) {
3851                    Ok(identity) => identity,
3852                    Err(_) => {
3853                        return Ok(vec![control_error_frame(
3854                            &frame,
3855                            ACTIVITY_MALFORMED,
3856                            "malformed activity read",
3857                        )?])
3858                    }
3859                };
3860                self.route_open_principal(&frame, identity.consumer_identity)?
3861            }
3862        };
3863        let admitted = matches!(principal, Ok(Principal::Direct))
3864            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3865        if !admitted {
3866            return Ok(vec![control_error_frame(
3867                &frame,
3868                ACTIVITY_READ_FORBIDDEN,
3869                "activity reads require an operator or trusted mirror principal",
3870            )?]);
3871        }
3872        let Some(ledger) = &self.activity else {
3873            return Ok(vec![control_error_frame(
3874                &frame,
3875                ACTIVITY_UNAVAILABLE,
3876                "activity ledger unavailable",
3877            )?]);
3878        };
3879        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3880            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3881        {
3882            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3883                Ok(request) => ledger.query(request).await,
3884                Err(_) => Err(ACTIVITY_MALFORMED),
3885            };
3886            return match result {
3887                Ok(reply) => Ok(vec![control_response_body_frame(
3888                    &frame,
3889                    &reply,
3890                    ops::ACTIVITY_QUERY,
3891                )?]),
3892                Err(code) => Ok(vec![control_error_frame(
3893                    &frame,
3894                    code,
3895                    "activity query refused",
3896                )?]),
3897            };
3898        }
3899        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3900            Ok(request) => request,
3901            Err(_) => {
3902                return Ok(vec![control_error_frame(
3903                    &frame,
3904                    ACTIVITY_MALFORMED,
3905                    "malformed activity subscription",
3906                )?])
3907            }
3908        };
3909        let mut subscription = match ledger.subscribe(request.since).await {
3910            Ok(subscription) => subscription,
3911            Err(code) => {
3912                return Ok(vec![control_error_frame(
3913                    &frame,
3914                    code,
3915                    "activity subscription refused",
3916                )?])
3917            }
3918        };
3919        let key = (ctx.connection_id, frame.header.corr);
3920        let sink = ctx.egress.clone();
3921        let mut streams = self
3922            .activity_streams
3923            .lock()
3924            .unwrap_or_else(|p| p.into_inner());
3925        streams.retain(|_, task| !task.is_finished());
3926        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3927            return Ok(vec![control_error_frame(
3928                &frame,
3929                "duplicate_subscription",
3930                "activity subscription correlation id already in use",
3931            )?]);
3932        }
3933        let task = tokio::spawn(async move {
3934            loop {
3935                let entry = if let Some(entry) = subscription.replay.pop_front() {
3936                    Some(entry)
3937                } else {
3938                    subscription.receiver.recv().await
3939                };
3940                let Some(entry) = entry else {
3941                    break;
3942                };
3943                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3944                    return;
3945                };
3946                let Ok(next) = Frame::build_with_version(
3947                    response_version(&frame),
3948                    FrameType::StreamData,
3949                    control_flags(),
3950                    0,
3951                    0,
3952                    frame.header.corr,
3953                    body,
3954                ) else {
3955                    return;
3956                };
3957                if sink.send(next).await.is_err() {
3958                    return;
3959                }
3960            }
3961            // Dropping a full subscriber's sender drains its accepted entries
3962            // before the terminal error, preserving their order on the wire.
3963            if let Some(code) = subscription.terminal_code() {
3964                if let Ok(end) =
3965                    control_error_frame(&frame, code, "activity subscriber fell behind")
3966                {
3967                    let _ = sink.send(end).await;
3968                }
3969            }
3970        });
3971        streams.insert(key, task.abort_handle());
3972        Ok(Vec::new())
3973    }
3974
3975    async fn handle_activity_record(
3976        &self,
3977        ctx: &RouteCtx,
3978        frame: Frame,
3979    ) -> Result<Vec<Frame>, RouterError> {
3980        use subc_control::{
3981            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3982            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3983        };
3984        let result = async {
3985            let module = self
3986                .forwarding
3987                .module_id_for_connection(ctx.connection_id)
3988                .ok()
3989                .flatten()
3990                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3991            let request: ActivityRecordRequest =
3992                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3993            crate::activity::validate(&module, &request)?;
3994            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3995            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3996            let at_ms = std::time::SystemTime::now()
3997                .duration_since(std::time::UNIX_EPOCH)
3998                .unwrap_or_default()
3999                .as_millis() as u64;
4000            ledger.record(module, request, actor, at_ms).await
4001        }
4002        .await;
4003        match result {
4004            Ok(entry) => Ok(vec![control_response_body_frame(
4005                &frame,
4006                &ActivityRecordResponse { seq: entry.seq },
4007                "activity.record",
4008            )?]),
4009            Err(code) => Ok(vec![control_error_frame(
4010                &frame,
4011                code,
4012                "activity recording refused",
4013            )?]),
4014        }
4015    }
4016
4017    fn activity_actor(
4018        &self,
4019        connection: ConnectionId,
4020        module: &str,
4021        on: Option<&subc_control::ActivityOn>,
4022    ) -> Result<subc_control::ActivityActor, &'static str> {
4023        use crate::forwarding::{DataRoute, DataRouteState};
4024        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4025        let principal = |id: &str| Principal::Reserved {
4026            module_id: id.to_owned(),
4027        };
4028        let mut actor = ActivityActor {
4029            principal: principal(module),
4030            scope_owner: None,
4031            agent_id: None,
4032            run_id: None,
4033            flow_id: None,
4034            project_root: None,
4035            workdir: None,
4036            project_id: None,
4037        };
4038        let scope = match on {
4039            None => return Ok(actor),
4040            Some(ActivityOn::Scope(scope)) => {
4041                let selector = subc_protocol::scope::ScopeSelector::new(
4042                    principal(module),
4043                    scope.scope_ref.clone(),
4044                    Some(scope.epoch),
4045                );
4046                self.scopes
4047                    .read()
4048                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4049                    .attribute(&principal(module), module, &selector, true)
4050                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4051            }
4052            Some(ActivityOn::Route(route)) => {
4053                let DataRoute::Module(DataRouteState::Bound(binding)) = self
4054                    .forwarding
4055                    .lookup_data_route(connection, route.channel, route.epoch)
4056                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4057                else {
4058                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4059                };
4060                if binding.module_id != module {
4061                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4062                }
4063                actor.principal = binding.principal.clone();
4064                actor.project_root = binding
4065                    .project_root
4066                    .as_ref()
4067                    .map(|root| root.as_path().to_path_buf());
4068                let Some(bound) = &binding.scope else {
4069                    return Ok(actor);
4070                };
4071                let owner = principal(&bound.owner);
4072                let selector = subc_protocol::scope::ScopeSelector::new(
4073                    owner.clone(),
4074                    bound.scope_ref.clone(),
4075                    Some(bound.tag.scope_epoch),
4076                );
4077                let admitted = self
4078                    .scopes
4079                    .read()
4080                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4081                    .attribute(&owner, module, &selector, true)
4082                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4083                if admitted.tag != bound.tag {
4084                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4085                }
4086                admitted
4087            }
4088        };
4089        actor.scope_owner = Some(scope.stamp.owner);
4090        actor.agent_id = scope.stamp.attributes.agent_id;
4091        actor.run_id = scope.stamp.attributes.run_id;
4092        actor.flow_id = scope.stamp.attributes.flow_id;
4093        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4094        actor.project_id = scope.stamp.project_id;
4095        Ok(actor)
4096    }
4097
4098    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4099        self.refresh_capability_requirements();
4100        // A bare connection count is ambiguous between many clients holding a
4101        // route each and one client accumulating hundreds, so publish the
4102        // concentration alongside it. Route state is best-effort here: a
4103        // diagnostic endpoint must still answer if the forwarding lock is
4104        // contended.
4105        let mut counters = self.counters.snapshot();
4106        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4107            obj.insert(
4108                "activity_unparseable_lines".into(),
4109                ledger.read_errors().into(),
4110            );
4111            obj.insert(
4112                "activity_lifecycle_append_failures".into(),
4113                ledger.lifecycle_failures().into(),
4114            );
4115            obj.insert(
4116                "activity_started_digest_skips".into(),
4117                ledger.lifecycle_digest_skips().into(),
4118            );
4119        }
4120        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4121            self.forwarding.client_route_concentration(),
4122            counters.as_object_mut(),
4123        ) {
4124            obj.insert(
4125                "client_connections_with_routes".into(),
4126                connections_with_routes.into(),
4127            );
4128            obj.insert("max_routes_on_one_connection".into(), max.into());
4129        }
4130        // A module that is being fast-refused and a module that is fine look
4131        // identical from a client that retries and succeeds, so name the open
4132        // breakers here. This rides the existing free-form counters object
4133        // rather than a new wire field, so no sibling that deserializes
4134        // `ServerDescribe` has to be rebuilt to keep reading it.
4135        if let (Some(open_breakers), Some(obj)) = (
4136            self.route_bind_breakers.open_snapshot(),
4137            counters.as_object_mut(),
4138        ) {
4139            obj.insert("route_bind_breakers_open".into(), open_breakers);
4140        }
4141        // A daemon whose local endpoint could not be bound keeps serving TCP,
4142        // and the connection file names no endpoint. Name the reason here, in
4143        // the free-form counters object, so an operator reading daemon facts
4144        // sees it without searching the log, and no wire field changes.
4145        if let (Some(reason), Some(obj)) = (
4146            self.local_endpoint_unavailable.clone(),
4147            counters.as_object_mut(),
4148        ) {
4149            obj.insert("local_endpoint_unavailable".into(), reason.into());
4150        }
4151        let response = ClientControlResponse::ServerDescribe {
4152            protocol_ver: PROTOCOL_VERSION,
4153            subc_ops: subc_ops(),
4154            capabilities: self.describe_capabilities(),
4155            connected_clients: self.connected_clients.count(),
4156            counters: Some(counters),
4157            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4158            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4159            capability_requirements: self.capability_requirement_statuses(),
4160            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4161            config_path: self
4162                .rescan
4163                .as_ref()
4164                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4165        };
4166        Ok(vec![control_response_body_frame(
4167            &frame,
4168            &response,
4169            "ClientControlResponse::ServerDescribe",
4170        )?])
4171    }
4172
4173    fn handle_catalog_list(
4174        &self,
4175        frame: Frame,
4176        module_id: Option<String>,
4177    ) -> Result<Vec<Frame>, RouterError> {
4178        let (generation, modules) = self
4179            .registry
4180            .list_modules_with_nonce_verification()
4181            .map_err(|err| {
4182                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4183            })?;
4184        let entries = modules
4185            .into_iter()
4186            .filter(|(registration, _)| {
4187                module_id
4188                    .as_deref()
4189                    .map(|wanted| registration.manifest.module_id == wanted)
4190                    .unwrap_or(true)
4191            })
4192            .map(|(registration, nonce_verified)| {
4193                let not_ready = self.not_ready_reason(&registration);
4194                let roles = registration.manifest.provides;
4195                CatalogEntry::new(
4196                    registration.manifest.module_id,
4197                    roles,
4198                    registration.control_ops,
4199                )
4200                .with_ready(not_ready.is_none())
4201                .with_not_ready(not_ready)
4202                .with_module_version(Some(registration.manifest.module_version))
4203                .with_capabilities(registration.manifest.capabilities)
4204                .with_self_signals(registration.manifest.self_signals)
4205                .with_events(registration.manifest.events)
4206                .with_operation_delivery(registration.manifest.operation_delivery)
4207                .with_operation_payload(registration.manifest.operation_payload)
4208                .with_flow_operations(registration.manifest.flow_operations)
4209                .with_confirm_operations(registration.manifest.confirm_operations)
4210                .with_rich_ui(registration.manifest.rich_ui)
4211                .with_role_declarations(registration.manifest.role_declarations)
4212                .with_nonce_verified(nonce_verified)
4213            })
4214            .collect();
4215        let response = ClientControlResponse::CatalogList {
4216            generation,
4217            modules: entries,
4218            subc_ops: subc_ops(),
4219        };
4220        Ok(vec![control_response_body_frame(
4221            &frame,
4222            &response,
4223            "ClientControlResponse::CatalogList",
4224        )?])
4225    }
4226
4227    /// The principal a route.open's `consumer_identity` attests: `Direct`
4228    /// when it presents none, `Reserved` when its launch nonce verifies, and
4229    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4230    /// can run before any refusal check without changing which check refuses.
4231    fn attest_route_open_consumer(
4232        &self,
4233        consumer_identity: Option<&ConsumerIdentity>,
4234    ) -> Option<Principal> {
4235        let Some(consumer_identity) = consumer_identity else {
4236            return Some(Principal::Direct);
4237        };
4238        self.supervisor
4239            .spawned_consumer_authorized(
4240                &consumer_identity.module_id,
4241                &consumer_identity.launch_nonce,
4242            )
4243            .then(|| Principal::Reserved {
4244                module_id: consumer_identity.module_id.clone(),
4245            })
4246    }
4247
4248    /// The caller fields for a refusal that only has the raw request frame,
4249    /// such as the connection-level admission refusal sent before the request
4250    /// reaches [`Self::handle_route_open`].
4251    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4252        let Ok(ClientControlRequest::RouteOpen {
4253            identity,
4254            consumer_identity,
4255            scope,
4256            ..
4257        }) = parse_client_control_request(&frame.body)
4258        else {
4259            return RouteOpenCaller::unknown();
4260        };
4261        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4262        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4263    }
4264
4265    fn route_open_principal(
4266        &self,
4267        frame: &Frame,
4268        consumer_identity: Option<ConsumerIdentity>,
4269    ) -> Result<Result<Principal, Frame>, RouterError> {
4270        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4271        self.route_open_principal_attested(frame, consumer_identity, attested)
4272    }
4273
4274    /// [`Self::route_open_principal`] with `attested` already taken by
4275    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4276    /// open checks the launch nonce once.
4277    fn route_open_principal_attested(
4278        &self,
4279        frame: &Frame,
4280        consumer_identity: Option<ConsumerIdentity>,
4281        attested: Option<Principal>,
4282    ) -> Result<Result<Principal, Frame>, RouterError> {
4283        if let Some(principal) = attested {
4284            return Ok(Ok(principal));
4285        }
4286        let Some(consumer_identity) = consumer_identity else {
4287            return Ok(Ok(Principal::Direct));
4288        };
4289
4290        Ok(Err(control_error_frame(
4291            frame,
4292            "bad_consumer_identity",
4293            format!(
4294                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4295                consumer_identity.module_id
4296            ),
4297        )?))
4298    }
4299
4300    /// Ordinary `route.open` refusals go through here; admission and breaker
4301    /// refusals log separately with their capacity or breaker state. The daemon can
4302    /// attest which code it sent: without the event, a client's "the daemon
4303    /// refused me" and the daemon's own view could only be reconciled by
4304    /// argument. Malformed input (`invalid_project_root`) does not come here;
4305    /// rejecting a request that was never a valid open is not a refusal of one.
4306    #[allow(clippy::too_many_arguments)]
4307    fn route_open_refusal_frame(
4308        &self,
4309        ctx: &RouteCtx,
4310        caller: &RouteOpenCaller,
4311        frame: &Frame,
4312        module_id: &str,
4313        reason: &'static str,
4314        code: &'static str,
4315        message: impl Into<String>,
4316    ) -> Result<Frame, RouterError> {
4317        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4318        control_error_frame(frame, code, message.into())
4319    }
4320
4321    /// Answer a waiting `route.open` whose relayed bind was refused.
4322    ///
4323    /// `daemon_authored` is provenance, not a code match: the daemon
4324    /// synthesized the refusal itself (a drain, a late reply, a scope that
4325    /// ended mid-bind) or the module refused the bind with its own body. A
4326    /// daemon refusal carries the daemon-origin bit; a module's own
4327    /// rejection is relayed with the bit clear, so a module that refuses
4328    /// with `module_reloading` still cannot mint a daemon refusal.
4329    #[allow(clippy::too_many_arguments)]
4330    fn route_bind_rejection(
4331        &self,
4332        ctx: &RouteCtx,
4333        caller: &RouteOpenCaller,
4334        frame: &Frame,
4335        target_module_id: &str,
4336        body: ErrorBody,
4337        daemon_authored: bool,
4338    ) -> Result<Vec<Frame>, RouterError> {
4339        // The daemon's own commit re-check refused the bind because the
4340        // scope ended or changed after admission. The module accepted;
4341        // counting it as a module rejection would blame the module. This
4342        // branch only runs for daemon-synthesized bodies: a module that
4343        // refuses with a scope code is answered as a module rejection below.
4344        if daemon_authored {
4345            let scope_code = match body.code.as_str() {
4346                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4347                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4348                _ => None,
4349            };
4350            if let Some(code) = scope_code {
4351                self.observe_route_open_refusal(
4352                    ctx,
4353                    caller,
4354                    target_module_id,
4355                    "scope_changed_before_commit",
4356                    code,
4357                );
4358                return Ok(vec![control_error_body_frame(frame, body)?]);
4359            }
4360        }
4361        self.counters
4362            .increment_route_open_refused("module_rejected");
4363        info!(
4364            target: "control",
4365            code = "module_rejected",
4366            module_code = ?body.code,
4367            module_id = ?target_module_id,
4368            connection_id = ctx.connection_id.get(),
4369            principal = %caller.principal,
4370            scope = %caller.scope,
4371            harness = %caller.harness,
4372            "route.open refused"
4373        );
4374        let refusal = if daemon_authored {
4375            control_error_body_frame(frame, body)?
4376        } else {
4377            control_relayed_error_body_frame(frame, body)?
4378        };
4379        Ok(vec![refusal])
4380    }
4381
4382    /// Refuse a `route.open` because the target module's bind-relay breaker is
4383    /// open, without attempting the relay.
4384    ///
4385    /// The wire code is `module_timeout`, which is the truth (the module has
4386    /// not been answering binds) and which both SDKs already classify as
4387    /// retryable with capped backoff. Reusing it is what keeps this change out
4388    /// of both SDKs; the daemon-side distinction lives in the counter key
4389    /// instead.
4390    ///
4391    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4392    /// While a breaker is open this fires on every open to that module, and the
4393    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4394    /// produced 261 lines about a single module inside 3000 lines of daemon
4395    /// log. The rare transitions are logged at warn/info instead and the volume
4396    /// is carried by the counter, so the evidence survives without the flood.
4397    /// The debug line keeps a per-refusal record reachable for whoever turns
4398    /// the level up.
4399    #[allow(clippy::too_many_arguments)]
4400    fn route_open_breaker_refusal_frame(
4401        &self,
4402        ctx: &RouteCtx,
4403        caller: &RouteOpenCaller,
4404        frame: &Frame,
4405        module_id: &str,
4406        consecutive_timeouts: u32,
4407        retry_in: Duration,
4408        probe_in_flight: bool,
4409    ) -> Result<Frame, RouterError> {
4410        self.counters
4411            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4412        debug!(
4413            target: "control",
4414            code = "module_timeout",
4415            module_id = ?module_id,
4416            connection_id = ctx.connection_id.get(),
4417            principal = %caller.principal,
4418            scope = %caller.scope,
4419            harness = %caller.harness,
4420            consecutive_timeouts,
4421            retry_in_ms = retry_in.as_millis() as u64,
4422            probe_in_flight,
4423            "route.open refused by open bind-relay breaker"
4424        );
4425        // Say what a caller can act on. An open bind-relay breaker means the
4426        // module timed out accepting several new routes in a row. The module
4427        // is still running and its established routes keep working; only new
4428        // route.open requests are refused until the cooldown ends and one
4429        // test route (the probe) gets through. A message that only counts
4430        // failed relays reads as "the module is down" to a worker that sees it.
4431        let detail = if probe_in_flight {
4432            "one test route is already being tried; retry once it settles".to_string()
4433        } else {
4434            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4435        };
4436        control_error_frame(
4437            frame,
4438            "module_timeout",
4439            format!(
4440                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4441                 timed out in a row); {detail}; its established routes are unaffected"
4442            ),
4443        )
4444    }
4445
4446    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4447    /// requester's bytes (an unknown target is whatever the client sent) and
4448    /// is Debug-formatted so control characters land in the log escaped
4449    /// rather than as terminal sequences for whoever tails it.
4450    ///
4451    /// `reason` names the check that refused, because one wire code has
4452    /// several senders: after a module registers, `target_unavailable` can
4453    /// come from a missing role, an inactive registration, a supervisor that
4454    /// has not marked the process live, a missing forwarding connection, or a
4455    /// failed relay, and a log that records only the code cannot say which of
4456    /// them fired. It is a static, daemon-chosen label per branch, so it is
4457    /// safe to print plainly and stays a closed set.
4458    fn observe_route_open_refusal(
4459        &self,
4460        ctx: &RouteCtx,
4461        caller: &RouteOpenCaller,
4462        module_id: &str,
4463        reason: &'static str,
4464        code: &'static str,
4465    ) {
4466        self.counters.increment_route_open_refused(code);
4467        info!(
4468            target: "control",
4469            code,
4470            reason,
4471            module_id = ?module_id,
4472            connection_id = ctx.connection_id.get(),
4473            principal = %caller.principal,
4474            scope = %caller.scope,
4475            harness = %caller.harness,
4476            "route.open refused"
4477        );
4478        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4479            self.route_outages.record_not_serving(module_id, reason);
4480        }
4481    }
4482
4483    /// Record an ACCEPTED route.open.
4484    ///
4485    /// Refusals have been logged and counted since the attestation work; accepts
4486    /// were invisible, so the daemon knew every principal it stamped and wrote
4487    /// none of them down. The party that attests the identity was the only party
4488    /// not recording it, which left a credential vault unable to name the sender
4489    /// of a call that reached it (claustrum #43) and left the launch-nonce
4490    /// concurrency question unanswerable from the outside.
4491    ///
4492    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4493    /// `code`/`module_id`/`connection_id` returns both directions of the same
4494    /// decision rather than two shapes a reader has to join by hand.
4495    ///
4496    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4497    /// and the difference carries information rather than being an
4498    /// inconsistency. This line is only reachable after a successful bind to a
4499    /// REGISTERED module, so the value has already passed HELLO validation
4500    /// including the path-hazard refusal and cannot contain control bytes. A
4501    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4502    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4503    ///
4504    /// Bare is also what every other daemon line already emits (`module
4505    /// registered`, `configured module supervised`). Shipping `?module_id` here
4506    /// made this instrument the only one in the file whose ids did not answer
4507    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4508    /// line whose whole purpose is being grepped beside its sibling.
4509    ///
4510    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4511    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4512    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4513    /// one are recorded identically. A test written against that harness passes
4514    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4515    /// green assertion that cannot fail. The same limit applies to the escaping
4516    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4517    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4518    /// Fencing either needs the real formatter, not the capture layer.
4519    ///
4520    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4521    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4522    /// to record. The ephemeral port would decay within minutes and answer only a
4523    /// live question. The identity question is instead answered by counting
4524    /// distinct live connections presenting one module's `consumer_identity` --
4525    /// "is anyone else holding this secret" rather than "is this the right
4526    /// process".
4527    fn observe_route_open_accept(
4528        &self,
4529        ctx: &RouteCtx,
4530        module_id: &str,
4531        principal: &str,
4532        scope: &str,
4533    ) {
4534        self.route_outages.record_accepted(module_id);
4535        self.counters.increment_route_open_accepted(principal);
4536        info!(
4537            target: "control",
4538            principal,
4539            module_id,
4540            connection_id = ctx.connection_id.get(),
4541            scope = %scope,
4542            "route.open accepted"
4543        );
4544    }
4545
4546    fn supervised_absent_route_open_refusal_frame(
4547        &self,
4548        ctx: &RouteCtx,
4549        caller: &RouteOpenCaller,
4550        frame: &Frame,
4551        module_id: &str,
4552        code: &'static str,
4553        status: &crate::supervise::ModuleStatus,
4554    ) -> Result<Frame, RouterError> {
4555        self.counters.increment_route_open_refused(code);
4556        info!(
4557            target: "control",
4558            code,
4559            reason = "supervised_not_registered",
4560            module_id = ?module_id,
4561            connection_id = ctx.connection_id.get(),
4562            principal = %caller.principal,
4563            scope = %caller.scope,
4564            harness = %caller.harness,
4565            state = %status.state,
4566            enabled = status.enabled,
4567            live = status.live,
4568            "route.open refused"
4569        );
4570        // A supervised module whose process has not registered is not
4571        // serving, whatever the reason; the supervisor knows this id, so it is
4572        // safe to track.
4573        self.route_outages
4574            .record_not_serving(module_id, "supervised_not_registered");
4575        let message = if let Some(pid) = status.waiting_for_previous_pid {
4576            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4577                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4578            })
4579        } else {
4580            format!(
4581                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4582                status.state, status.enabled, status.live
4583            )
4584        };
4585        control_error_frame(frame, code, message)
4586    }
4587
4588    async fn handle_route_open(
4589        &self,
4590        ctx: &RouteCtx,
4591        frame: Frame,
4592        request: RouteOpenRequest,
4593        timings: &mut RouteOpenTimings,
4594    ) -> Result<Vec<Frame>, RouterError> {
4595        let RouteOpenRequest {
4596            target,
4597            mut identity,
4598            consumer_identity,
4599            consumer_capabilities,
4600            role_versions,
4601            admission_facts,
4602            scope,
4603        } = request;
4604        let target_module_id = target_module_id(&target).to_string();
4605        timings.module_id.clone_from(&target_module_id);
4606        if self
4607            .registry
4608            .get_module_by_connection(ctx.connection_id)
4609            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4610            .is_some()
4611        {
4612            return Ok(vec![control_error_frame(
4613                &frame,
4614                "invalid_request",
4615                "module connections cannot open client routes",
4616            )?]);
4617        }
4618        debug!(
4619            connection_id = ctx.connection_id.get(),
4620            corr = frame.header.corr,
4621            module_id = %target_module_id,
4622            "handling route.open"
4623        );
4624
4625        // The caller's attested principal, requested scope and harness, for
4626        // every `route.open refused` line below. The launch nonce is checked
4627        // here, once, and the answer is reused where the principal is
4628        // enforced further down, so the order of refusals is unchanged.
4629        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4630        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4631
4632        // A malformed declaration is refused first, before anything about the
4633        // target is looked up: the same body would be refused against any
4634        // module, so the caller learns nothing by retrying or waiting. An empty
4635        // map declares nothing and travels as no field at all, so a provider
4636        // only ever sees a missing field or a non-empty one.
4637        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4638        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4639            self.observe_route_open_refusal(
4640                ctx,
4641                &caller,
4642                &target_module_id,
4643                "invalid_role_versions",
4644                error_codes::INVALID_REQUEST,
4645            );
4646            return Ok(vec![control_error_body_frame(
4647                &frame,
4648                ErrorBody {
4649                    code: error_codes::INVALID_REQUEST.to_string(),
4650                    message: error.to_string(),
4651                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4652                },
4653            )?]);
4654        }
4655
4656        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4657            if status.waiting_for_previous_pid.is_some() {
4658                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4659                    ctx,
4660                    &caller,
4661                    &frame,
4662                    &target_module_id,
4663                    "module_warming",
4664                    &status,
4665                )?]);
4666            }
4667        }
4668
4669        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4670        // opposite. Below, a caller learns whether a module is unregistered,
4671        // supervised-but-down (with state/enabled/live), or registered without the
4672        // requested role. Elsewhere that is an enumeration leak: a probe learning
4673        // the shape of a fleet it cannot otherwise see.
4674        //
4675        // It is not one here, and the reason is the ACCESS MODEL rather than
4676        // anything about these errors. Reaching route.open requires the
4677        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4678        // connection file, so any caller who completes it already runs as this
4679        // user -- and can read subc.jsonc for the module list and `ck module
4680        // status` for live state. The reply discloses nothing the caller cannot
4681        // read more easily from disk, while the precision is load-bearing:
4682        // `unknown_module` is retryable and a missing role is not.
4683        //
4684        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4685        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4686        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4687        let Some(registration) = self
4688            .registry
4689            .get_module(&target_module_id)
4690            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4691        else {
4692            if let Some((status, warming)) =
4693                self.supervisor_status(&target_module_id, frame.header.corr)?
4694            {
4695                // BEFORE the two availability codes below, because for a module
4696                // that speaks no subc wire both of them are false comfort: they
4697                // say "not right now" and are retried, and this module will
4698                // never register no matter how long the caller waits. The
4699                // absence here is the declaration being honoured, not a module
4700                // that is late.
4701                if status.protocol == ModuleProtocol::None {
4702                    return Ok(vec![self.route_open_refusal_frame(
4703                        ctx,
4704                        &caller,
4705                        &frame,
4706                        &target_module_id,
4707                        "protocol_none",
4708                        error_codes::MODULE_NO_PROTOCOL,
4709                        format!(
4710                            "module_id '{target_module_id}' is declared protocol: none; \
4711                             it speaks no subc wire and serves no routes"
4712                        ),
4713                    )?]);
4714                }
4715                let code = if warming {
4716                    "module_warming"
4717                } else {
4718                    "target_unavailable"
4719                };
4720                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4721                    ctx,
4722                    &caller,
4723                    &frame,
4724                    &target_module_id,
4725                    code,
4726                    &status,
4727                )?]);
4728            }
4729            if let Some(removed_ago_ms) =
4730                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4731            {
4732                return Ok(vec![self.route_open_refusal_frame(
4733                    ctx,
4734                    &caller,
4735                    &frame,
4736                    &target_module_id,
4737                    "removed",
4738                    error_codes::MODULE_REMOVED,
4739                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4740                )?]);
4741            }
4742            return Ok(vec![self.route_open_refusal_frame(
4743                ctx,
4744                &caller,
4745                &frame,
4746                &target_module_id,
4747                "not_registered",
4748                error_codes::UNKNOWN_MODULE,
4749                format!("module_id '{target_module_id}' is not registered"),
4750            )?]);
4751        };
4752
4753        // Best-effort only: registry readiness and forwarding reservation use
4754        // different locks, so a module can flip readiness between this read and
4755        // the relay. Modules must still tolerate an `on_bind` while not ready.
4756        if !registration.ready {
4757            self.counters
4758                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4759            info!(
4760                target: "control",
4761                code = error_codes::MODULE_WARMING,
4762                module_id = ?target_module_id,
4763                connection_id = ctx.connection_id.get(),
4764                principal = %caller.principal,
4765                scope = %caller.scope,
4766                harness = %caller.harness,
4767                reason = "declared_not_ready",
4768                "route.open refused"
4769            );
4770            // The module is registered but says it cannot take work, which is
4771            // an outage from the caller's side even though its process is up.
4772            self.route_outages
4773                .record_not_serving(&target_module_id, "declared_not_ready");
4774            return Ok(vec![control_error_body_frame(
4775                &frame,
4776                ErrorBody {
4777                    code: error_codes::MODULE_WARMING.to_string(),
4778                    message: format!(
4779                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4780                    ),
4781                    detail: Some(serde_json::json!({
4782                        "reason": "declared_not_ready"
4783                    })),
4784                },
4785            )?]);
4786        }
4787
4788        // Effective readiness, second half: a module that declares a capability
4789        // `need: required` is not routable while that capability has no
4790        // registered provider. It is enforced HERE, as a retryable routing
4791        // refusal, and deliberately not as spawn ordering or a boot block. The
4792        // module is still started and registered and can make its own calls;
4793        // spawn ordering is a promise that cannot be kept once a provider
4794        // crashes at runtime, and refusing to boot would stop the whole
4795        // machine, including the tools needed to fix its configuration.
4796        //
4797        // "Provided" is the evaluator's verdict, which counts a provider as
4798        // soon as it has REGISTERED, not once it is ready. Two modules that
4799        // require each other's capabilities are therefore both routable once
4800        // both register; counting readiness instead would deadlock them.
4801        //
4802        // Only new opens are refused. Routes already bound when a provider
4803        // goes away stay bound: nothing here tears them down, and the module
4804        // answers them as it can. Like the readiness read above this is
4805        // best-effort against a provider registering or leaving concurrently.
4806        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4807            self.counters
4808                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4809            info!(
4810                target: "control",
4811                code = error_codes::MODULE_WARMING,
4812                module_id = ?target_module_id,
4813                connection_id = ctx.connection_id.get(),
4814                principal = %caller.principal,
4815                scope = %caller.scope,
4816                harness = %caller.harness,
4817                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4818                capability = %capability,
4819                "route.open refused"
4820            );
4821            return Ok(vec![control_error_body_frame(
4822                &frame,
4823                ErrorBody {
4824                    code: error_codes::MODULE_WARMING.to_string(),
4825                    message: format!(
4826                        "module_id '{target_module_id}' requires capability '{capability}', \
4827                         which no registered module provides; retry"
4828                    ),
4829                    detail: Some(serde_json::json!({
4830                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4831                        "capability": capability,
4832                    })),
4833                },
4834            )?]);
4835        }
4836
4837        if !target_has_required_role(&target, &registration.manifest.provides) {
4838            return Ok(vec![self.route_open_refusal_frame(
4839                ctx,
4840                &caller,
4841                &frame,
4842                &target_module_id,
4843                "role_not_provided",
4844                "target_unavailable",
4845                format!("module_id '{target_module_id}' does not provide the requested target"),
4846            )?]);
4847        }
4848
4849        if registration.state != ChannelState::Active {
4850            return Ok(vec![self.route_open_refusal_frame(
4851                ctx,
4852                &caller,
4853                &frame,
4854                &target_module_id,
4855                "registration_not_active",
4856                "target_unavailable",
4857                format!("module_id '{target_module_id}' is not active"),
4858            )?]);
4859        }
4860
4861        if self
4862            .forwarding
4863            .module_is_draining(&target_module_id)
4864            .map_err(RouterError::Forwarding)?
4865        {
4866            return Ok(vec![self.route_open_refusal_frame(
4867                ctx,
4868                &caller,
4869                &frame,
4870                &target_module_id,
4871                "reloading",
4872                "module_reloading",
4873                format!("module_id '{target_module_id}' is reloading"),
4874            )?]);
4875        }
4876
4877        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4878            process_liveness.process_live(&target_module_id) == Some(false)
4879        }) {
4880            // A module the supervisor is restarting or reloading can still hold
4881            // a registration: the old process before its connection closes, or
4882            // a new one that registered while the supervisor was draining. The
4883            // forwarding table does not see that as draining, but the consumer
4884            // should still be told to retry soon, exactly as for the drain
4885            // above, rather than that the target is unavailable.
4886            if process_liveness.process_replacing(&target_module_id) {
4887                return Ok(vec![self.route_open_refusal_frame(
4888                    ctx,
4889                    &caller,
4890                    &frame,
4891                    &target_module_id,
4892                    "reloading",
4893                    "module_reloading",
4894                    format!("module_id '{target_module_id}' is reloading"),
4895                )?]);
4896            }
4897            return Ok(vec![self.route_open_refusal_frame(
4898                ctx,
4899                &caller,
4900                &frame,
4901                &target_module_id,
4902                "supervisor_not_live",
4903                "target_unavailable",
4904                format!("module_id '{target_module_id}' is not live"),
4905            )?]);
4906        }
4907
4908        if !self
4909            .forwarding
4910            .has_live_module_connection(&target_module_id)
4911            .map_err(RouterError::Forwarding)?
4912        {
4913            return Ok(vec![self.route_open_refusal_frame(
4914                ctx,
4915                &caller,
4916                &frame,
4917                &target_module_id,
4918                "no_forwarding_connection",
4919                "target_unavailable",
4920                format!("module_id '{target_module_id}' has no live forwarding connection"),
4921            )?]);
4922        }
4923
4924        if let Some(error) =
4925            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4926        {
4927            self.observe_route_open_refusal(
4928                ctx,
4929                &caller,
4930                &target_module_id,
4931                "op_not_allowed",
4932                "op_not_allowed",
4933            );
4934            return Ok(vec![error]);
4935        }
4936
4937        let principal =
4938            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4939                Ok(principal) => principal,
4940                Err(error) => {
4941                    self.observe_route_open_refusal(
4942                        ctx,
4943                        &caller,
4944                        &target_module_id,
4945                        "bad_consumer_identity",
4946                        "bad_consumer_identity",
4947                    );
4948                    return Ok(vec![error]);
4949                }
4950            };
4951        timings.principal = principal_label(&principal);
4952
4953        // Only Callosum can attest a production phone build. Consume the claim
4954        // here so it can be kept on the route without reaching any module.
4955        let build_class = identity.build_class.take().filter(|build_class| {
4956            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
4957                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
4958            } else {
4959                if self
4960                    .build_class_warned
4961                    .lock()
4962                    .unwrap_or_else(|p| p.into_inner())
4963                    .insert(ctx.connection_id)
4964                {
4965                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
4966                        "ignoring build_class from a non-Callosum opener");
4967                }
4968                false
4969            }
4970        });
4971
4972        // This is attested, control-plane policy for supervised module origins.
4973        // Keep it before route reservation and out of the opaque forwarding hot
4974        // path: data frames must never acquire a per-frame capability check.
4975        if let Principal::Reserved {
4976            module_id: opening_module_id,
4977        } = &principal
4978        {
4979            if let Some(opening_registration) = self
4980                .registry
4981                .get_module(opening_module_id)
4982                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4983            {
4984                if let Some(capability) =
4985                    denied_capability(&opening_registration.manifest, &registration.manifest)
4986                {
4987                    warn!(
4988                        opening_module_id,
4989                        target_module_id,
4990                        capability,
4991                        "refusing route.open because an attested capability deny edge matches"
4992                    );
4993                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4994                        &frame,
4995                        &target_module_id,
4996                        "capability_deny_edge",
4997                        "capability_forbidden",
4998                        format!(
4999                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5000                        ),
5001                    )?]);
5002                }
5003            }
5004        }
5005
5006        if admission_facts.is_some() {
5007            let carrier_matches = matches!(
5008                &principal,
5009                Principal::Reserved { module_id }
5010                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5011            );
5012            if !carrier_matches {
5013                return Ok(vec![self.route_open_refusal_frame(
5014                    ctx,
5015                    &caller,
5016                    &frame,
5017                    &target_module_id,
5018                    "admission_facts_carrier_not_permitted",
5019                    "admission_facts_not_permitted",
5020                    "admission facts may only be carried by the configured reserved module",
5021                )?]);
5022            }
5023
5024            let target_allowed = self
5025                .admission_facts_targets
5026                .as_ref()
5027                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5028            if !target_allowed {
5029                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5030                    &frame,
5031                    &target_module_id,
5032                    "admission_facts_target_not_listed",
5033                    "admission_facts_target_not_allowed",
5034                    format!(
5035                        "admission facts are not permitted for target module_id '{target_module_id}'"
5036                    ),
5037                )?]);
5038            }
5039
5040            // Keep the value opaque to subc. The downstream admission validator owns
5041            // schema and semantic checks; this daemon only enforces carrier authority
5042            // and the configured destination allowlist.
5043        }
5044
5045        // Scope admission, on the attested principal above and never on the
5046        // request body. The tag read here travels with the pending bind and is
5047        // compared with the published one at commit, so a sync between here
5048        // and the module's ack refuses the open instead of binding a stamp
5049        // that is no longer true.
5050        let (bound_scope, mut scope_stamp) = match &scope {
5051            None => (None, None),
5052            Some(selector) => {
5053                let owner_configured = match &selector.owner {
5054                    // A reserved owner counts as configured from before its
5055                    // process is spawned (see `SupervisorHandle::is_configured`).
5056                    // So an owner that has not synced its scopes yet is refused
5057                    // as retryable (`scope_not_synced`), not as one that will
5058                    // never sync.
5059                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5060                    _ => false,
5061                };
5062                let mut scopes = self
5063                    .scopes
5064                    .write()
5065                    .unwrap_or_else(|poisoned| poisoned.into_inner());
5066                let admitted = scopes
5067                    .admit(&principal, &target_module_id, selector, owner_configured)
5068                    .and_then(|admission| {
5069                        crate::scopes::check_target_flow_support(
5070                            &admission.stamp,
5071                            &target_module_id,
5072                            registration.manifest.capabilities.as_ref(),
5073                        )?;
5074                        crate::scopes::check_target_agent_run_support(
5075                            &admission.stamp,
5076                            &target_module_id,
5077                            registration.manifest.capabilities.as_ref(),
5078                        )?;
5079                        let bound = scopes.reserve_bind(
5080                            &admission,
5081                            &principal,
5082                            ctx.connection_id,
5083                            selector.hold_presence,
5084                        );
5085                        Ok((admission, bound))
5086                    });
5087                match admitted {
5088                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5089                    Err(refusal) => {
5090                        return Ok(vec![self.route_open_refusal_frame(
5091                            ctx,
5092                            &caller,
5093                            &frame,
5094                            &target_module_id,
5095                            refusal.code,
5096                            refusal.code,
5097                            refusal.message,
5098                        )?]);
5099                    }
5100                }
5101            }
5102        };
5103        // From here on a refusal names the scope by the stamp admission
5104        // issued, exactly as the `route.open accepted` line does.
5105        if let Some(stamp) = &scope_stamp {
5106            caller.scope = route_open_scope_label(stamp);
5107        }
5108
5109        // Bind admits a root that no longer exists on disk, because refusing here
5110        // closes the only exit from a paused run: cancel needs a bound route, and a
5111        // renamed or reclaimed directory makes that route unopenable forever. The
5112        // run itself is intact and still addressable by its recorded identity.
5113        //
5114        // This does NOT relax the rule the strict constructor protects. That rule is
5115        // that no root is ever aliased into NEW durable state -- a missing component
5116        // can reappear as a symlink elsewhere, which would move the identity and
5117        // split a session's history across two of them. The engine now refuses the
5118        // two operations that create such state (send and import) at admission,
5119        // which is a narrower way to hold the same invariant: reads and terminations
5120        // are admitted, writes are not. That refusal had to ship before this line
5121        // changed, or there is an interval where a send commits under a provisional
5122        // identity -- the exact failure the original policy existed to prevent.
5123        //
5124        // Resolution follows realpath rather than lexical cleanup: the longest
5125        // existing ancestor is canonicalized and the missing tail re-appended, so a
5126        // live root is unchanged and a vanished leaf keeps the identity it was
5127        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
5128        // same caller the moment the directory vanished, which strands the run more
5129        // quietly than refusing it.
5130        timings.enter(RouteOpenPhase::ProjectRoot);
5131        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
5132            Ok(project_root) => project_root,
5133            Err(err) => {
5134                return Ok(vec![control_error_frame(
5135                    &frame,
5136                    "invalid_project_root",
5137                    err.to_string(),
5138                )?])
5139            }
5140        };
5141        timings.enter(RouteOpenPhase::Admission);
5142        identity.project_root = project_root.as_path().to_path_buf();
5143
5144        // Last gate before any relay work, and deliberately after the cheap
5145        // registry and availability checks above: those name a more precise
5146        // condition (unknown, removed, reloading) and a caller is better served
5147        // by the precise code than by this one.
5148        //
5149        // Everything below this point costs an egress permit, a reserved handle
5150        // pair and, if the module does not answer, the whole relay budget. The
5151        // reader no longer waits for that budget, so cap each target explicitly;
5152        // serial dispatch used to provide the accidental cap of one relay per
5153        // connection. Admission is a mutex-protected count and never waits.
5154        let _concurrency_guard = match self
5155            .route_bind_concurrency
5156            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5157        {
5158            Ok(guard) => guard,
5159            Err(in_flight) => {
5160                return Ok(vec![self.route_open_target_capacity_refusal(
5161                    ctx,
5162                    &caller,
5163                    &frame,
5164                    &target_module_id,
5165                    in_flight,
5166                )?]);
5167            }
5168        };
5169
5170        // A module that has already burned the whole budget `threshold` times
5171        // in a row does not get to charge it again until a probe says it recovered.
5172        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5173            RouteBindAdmission::Admitted { guard, probe } => {
5174                if probe {
5175                    info!(
5176                        module_id = %target_module_id,
5177                        connection_id = ctx.connection_id.get(),
5178                        "route.bind breaker half-open: admitting one probe"
5179                    );
5180                }
5181                guard
5182            }
5183            RouteBindAdmission::Refused {
5184                consecutive_timeouts,
5185                retry_in,
5186                probe_in_flight,
5187            } => {
5188                return Ok(vec![self.route_open_breaker_refusal_frame(
5189                    ctx,
5190                    &caller,
5191                    &frame,
5192                    &target_module_id,
5193                    consecutive_timeouts,
5194                    retry_in,
5195                    probe_in_flight,
5196                )?]);
5197            }
5198        };
5199
5200        // Resolve the per-module budget here so the wait matches the operator's
5201        // intent for this specific target. A per-module override in
5202        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5203        // daemons) wins over the daemon-wide default.
5204        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5205        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5206        let operator_app = self
5207            .operator_apps
5208            .lock()
5209            .unwrap_or_else(|p| p.into_inner())
5210            .get(&ctx.connection_id)
5211            .cloned();
5212        timings.enter(RouteOpenPhase::EgressReservation);
5213        let pending = match self
5214            .forwarding
5215            .begin_route_bind_relay_for(
5216                ctx.connection_id,
5217                ctx.egress.clone(),
5218                response_version(&frame),
5219                frame.header.corr,
5220                &target_module_id,
5221                principal.clone(),
5222                bound_scope,
5223                Some(project_root),
5224                identity.harness.clone(),
5225                build_class,
5226                operator_app.clone(),
5227                relay_deadline,
5228            )
5229            .await
5230        {
5231            Ok(pending) => pending,
5232            Err(err) => {
5233                return Ok(vec![self.route_open_refusal_frame(
5234                    ctx,
5235                    &caller,
5236                    &frame,
5237                    &target_module_id,
5238                    "relay_reservation_failed",
5239                    forwarding_error_code(&err),
5240                    err.to_string(),
5241                )?])
5242            }
5243        };
5244        timings.enter(RouteOpenPhase::Admission);
5245        let crate::forwarding::PendingRouteBindRelay {
5246            endpoint,
5247            module_sink,
5248            negotiated_ver,
5249            client_channel,
5250            client_epoch,
5251            module_channel,
5252            module_epoch,
5253            corr: relay_corr,
5254            receiver,
5255        } = pending;
5256        let mut reservation =
5257            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5258
5259        // Reserving egress can wait while a module reconnects or a swap cuts
5260        // over. Check the relay's captured connection, not the earlier by-id
5261        // lookup: the original module's flow/run capabilities cannot authorize
5262        // its replacement. The captured sink stays bound to that connection.
5263        let relay_registration = self
5264            .registry
5265            .get_module_by_connection(endpoint.connection_id)
5266            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5267        let relay_capabilities = relay_registration
5268            .as_ref()
5269            .and_then(|registration| registration.manifest.capabilities.as_ref());
5270        if let Some(selector) = &scope {
5271            // Egress reservation may wait across a scope update. Recheck the
5272            // proof against the current record before sending the captured bind.
5273            if let Err(refusal) = self
5274                .scopes
5275                .read()
5276                .unwrap_or_else(|poisoned| poisoned.into_inner())
5277                .check_bind_proof(&principal, selector)
5278            {
5279                reservation.release_and_disarm();
5280                return Ok(vec![self.route_open_refusal_frame(
5281                    ctx,
5282                    &caller,
5283                    &frame,
5284                    &target_module_id,
5285                    refusal.code,
5286                    refusal.code,
5287                    refusal.message,
5288                )?]);
5289            }
5290        }
5291        if let Some(stamp) = scope_stamp
5292            .as_ref()
5293            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5294        {
5295            if let Err(refusal) = crate::scopes::check_target_flow_support(
5296                stamp,
5297                &target_module_id,
5298                relay_capabilities,
5299            )
5300            .and_then(|()| {
5301                crate::scopes::check_target_agent_run_support(
5302                    stamp,
5303                    &target_module_id,
5304                    relay_capabilities,
5305                )
5306            }) {
5307                reservation.release_and_disarm();
5308                return Ok(vec![self.route_open_refusal_frame(
5309                    ctx,
5310                    &caller,
5311                    &frame,
5312                    &target_module_id,
5313                    refusal.code,
5314                    refusal.code,
5315                    refusal.message,
5316                )?]);
5317            }
5318        }
5319
5320        // Older readers may reject unknown fields. Select optional fields from
5321        // the captured endpoint's declaration, including on unscoped binds.
5322        if !relay_capabilities.is_some_and(|caps| {
5323            caps.provides
5324                .iter()
5325                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5326        }) {
5327            if let Some(stamp) = scope_stamp.as_mut() {
5328                stamp.workdir = None;
5329                stamp.project_id = None;
5330            }
5331        }
5332        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5333        // attribution. The run gate above has already required the target to
5334        // declare `worker-runs/v1`, which promises to read both
5335        // `on_behalf_of_agent` and `call_mode`.
5336        let worker_run = scope_stamp
5337            .as_ref()
5338            .is_some_and(crate::scopes::is_worker_run)
5339            && relay_capabilities.is_some_and(|caps| {
5340                caps.provides
5341                    .iter()
5342                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5343            });
5344        if !worker_run
5345            && !relay_capabilities.is_some_and(|caps| {
5346                caps.provides
5347                    .iter()
5348                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5349            })
5350        {
5351            if let Some(stamp) = scope_stamp.as_mut() {
5352                stamp.attributes.on_behalf_of_agent = None;
5353            }
5354        }
5355        if !relay_capabilities.is_some_and(|caps| {
5356            caps.provides
5357                .iter()
5358                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5359        }) {
5360            if let Some(stamp) = scope_stamp.as_mut() {
5361                stamp.attributes.flow_version = None;
5362                stamp.attributes.flow_code_hash = None;
5363                stamp.attributes.flow_install_id = None;
5364            }
5365        }
5366        let call_mode = if worker_run
5367            || relay_capabilities
5368                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5369        {
5370            Some(derive_call_mode(scope_stamp.as_ref()))
5371        } else {
5372            if let Some(stamp) = scope_stamp.as_mut() {
5373                stamp.expires_at_ms = None;
5374            }
5375            None
5376        };
5377
5378        debug!(
5379            connection_id = ctx.connection_id.get(),
5380            client_channel,
5381            client_epoch,
5382            module_channel,
5383            module_epoch,
5384            "reserved route handle pair"
5385        );
5386        // Rendered BEFORE the move into the relay, because the accept arm below
5387        // is where it is logged and the principal is gone by then.
5388        let route_principal_label = principal_label(&principal);
5389        let relay = ModuleControlRequest::RouteBind {
5390            route_channel: module_channel,
5391            epoch: module_epoch,
5392            target,
5393            identity,
5394            principal: Some(principal),
5395            consumer_capabilities,
5396            role_versions,
5397            admission_facts,
5398            scope: scope_stamp,
5399            call_mode,
5400        };
5401        let peer = RouteBindPeer::new()
5402            .with_observed_peer(
5403                relay_capabilities
5404                    .is_some_and(|caps| {
5405                        caps.provides
5406                            .iter()
5407                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5408                    })
5409                    .then(|| {
5410                        self.observed_peers
5411                            .lock()
5412                            .unwrap_or_else(|p| p.into_inner())
5413                            .get(&ctx.connection_id)
5414                            .cloned()
5415                            .unwrap_or_else(ObservedPeer::unobserved)
5416                    }),
5417            )
5418            .with_operator_app(
5419                operator_app
5420                    .filter(|stamp| {
5421                        stamp.delivery_enabled(&self.operator_app_policy)
5422                            && relay_capabilities.is_some_and(|caps| {
5423                                caps.provides
5424                                    .iter()
5425                                    .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5426                            })
5427                    })
5428                    .map(|stamp| stamp.app),
5429            );
5430        let relay_body = encode_route_bind(&relay, &peer).map_err(|err| {
5431            RouterError::backend(
5432                0,
5433                frame.header.corr,
5434                format!("failed to encode route.bind request: {err}"),
5435            )
5436        })?;
5437        let relay_frame = Frame::build_with_version(
5438            negotiated_ver,
5439            FrameType::Request,
5440            control_flags(),
5441            0,
5442            0,
5443            relay_corr,
5444            relay_body,
5445        )
5446        .map_err(RouterError::FrameBuild)?;
5447
5448        timings.enter(RouteOpenPhase::BindSend);
5449        let sent = module_sink.send(relay_frame).await;
5450        timings.enter(RouteOpenPhase::Admission);
5451        if let Err(err) = sent {
5452            reservation.release_and_disarm();
5453            return Ok(vec![self.route_open_refusal_frame(
5454                ctx,
5455                &caller,
5456                &frame,
5457                &target_module_id,
5458                "relay_send_failed",
5459                "target_unavailable",
5460                err.to_string(),
5461            )?]);
5462        }
5463
5464        #[cfg(test)]
5465        if let Some(hold) = &self.route_bind_relay_sent_hold {
5466            // First wait: tell the test the bind is queued. Second wait: stay
5467            // here until the test has driven whatever it wants to happen
5468            // between queueing the bind and marking it enqueued.
5469            hold.wait().await;
5470            hold.wait().await;
5471        }
5472
5473        // A missing relay is not by itself an abandoned bind: the module may
5474        // already have acked it and the route may be committed. Only a relay
5475        // dropped without any module reply needs this GOODBYE; one a reply
5476        // settled has already had its GOODBYE decided by the reply path.
5477        if self
5478            .forwarding
5479            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5480            .map_err(RouterError::Forwarding)?
5481            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5482        {
5483            self.send_abandoned_route_bind_goodbye(
5484                &module_sink,
5485                negotiated_ver,
5486                module_channel,
5487                module_epoch,
5488            );
5489        }
5490
5491        timings.enter(RouteOpenPhase::BindAckWait);
5492        let outcome = timeout_at(relay_deadline, receiver).await;
5493        timings.enter(RouteOpenPhase::Admission);
5494        match outcome {
5495            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5496                reservation.disarm();
5497                if breaker.record_accepted() {
5498                    info!(
5499                        module_id = %target_module_id,
5500                        "route.bind breaker closed: the probe was accepted"
5501                    );
5502                }
5503                self.observe_route_open_accept(
5504                    ctx,
5505                    &target_module_id,
5506                    &route_principal_label,
5507                    &caller.scope,
5508                );
5509                Ok(Vec::new())
5510            }
5511            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5512                reservation.release_and_disarm();
5513                // A module that says no in microseconds is healthy. Rejection
5514                // is a different condition with its own refusal and must not
5515                // move the breaker.
5516                breaker.record_inconclusive();
5517                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5518            }
5519            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5520                reservation.release_and_disarm();
5521                breaker.record_inconclusive();
5522                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5523            }
5524            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5525                reservation.release_and_disarm();
5526                breaker.record_inconclusive();
5527                // Fires when the module's connection closes while a relayed
5528                // bind is pending -- typically a caller racing a module restart
5529                // whose bind was relayed BEFORE the drain mark went up. Logged
5530                // because the caller sees only its own error and the fleet has
5531                // already spent one diagnosis round unable to tell this arm
5532                // from a relay timeout without daemon-side evidence.
5533                tracing::warn!(
5534                    module_id = %target_module_id,
5535                    "route.bind relay abandoned: {message}"
5536                );
5537                Ok(vec![self.route_open_refusal_frame(
5538                    ctx,
5539                    &caller,
5540                    &frame,
5541                    &target_module_id,
5542                    "relay_abandoned",
5543                    "target_unavailable",
5544                    message,
5545                )?])
5546            }
5547            Ok(Err(_)) => {
5548                reservation.release_and_disarm();
5549                breaker.record_inconclusive();
5550                Ok(vec![self.route_open_refusal_frame(
5551                    ctx,
5552                    &caller,
5553                    &frame,
5554                    &target_module_id,
5555                    "relay_waiter_canceled",
5556                    "target_unavailable",
5557                    "route.bind relay waiter was canceled before the module responded",
5558                )?])
5559            }
5560            Err(_) => {
5561                reservation.release_and_disarm();
5562                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5563                // answer at all is the one condition a fast refusal can
5564                // usefully stand in for; every other arm already answered.
5565                if let Some(opened) = breaker.record_timeout(
5566                    self.route_bind_breaker_threshold,
5567                    self.route_bind_breaker_cooldown,
5568                ) {
5569                    warn!(
5570                        module_id = %target_module_id,
5571                        consecutive_timeouts = opened.consecutive_timeouts,
5572                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5573                        reopened_after_probe = opened.reopened_after_probe,
5574                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5575                    );
5576                }
5577                // The generous budget just burned to no answer: the module is
5578                // registered and its connection is up, but its bind handler sat
5579                // on the ack for the full budget (warm-on-bind, cold configure,
5580                // or a wedged handler). Every earlier unavailability shape
5581                // fast-refuses BEFORE the relay, so this arm firing means the
5582                // slowness is module-side -- log it so the per-module timeline
5583                // is reconstructable without client audit rows.
5584                tracing::warn!(
5585                    module_id = %target_module_id,
5586                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5587                    "route.bind relay timed out: module did not ack within budget"
5588                );
5589                Ok(vec![self.route_open_refusal_frame(
5590                    ctx,
5591                    &caller,
5592                    &frame,
5593                    &target_module_id,
5594                    "relay_timed_out",
5595                    "module_timeout",
5596                    format!(
5597                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5598                        route_bind_relay_timeout
5599                    ),
5600                )?])
5601            }
5602        }
5603    }
5604
5605    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5606        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5607            snapshot: self.supervisor.spawn_snapshot(),
5608        };
5609        Ok(vec![control_response_body_frame(
5610            &frame,
5611            &response,
5612            "ClientControlResponse::SupervisorSpawnSnapshot",
5613        )?])
5614    }
5615
5616    fn handle_supervisor_spawn_subscribe(
5617        &self,
5618        ctx: &RouteCtx,
5619        frame: Frame,
5620        since: Option<SpawnCursor>,
5621    ) -> Result<Vec<Frame>, RouterError> {
5622        match self.supervisor.subscribe_spawns(
5623            ctx.connection_id,
5624            frame.header.corr,
5625            response_version(&frame),
5626            since,
5627            ctx.egress.clone(),
5628        ) {
5629            Ok(()) => Ok(Vec::new()),
5630            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5631                Ok(vec![control_error_body_frame(
5632                    &frame,
5633                    ErrorBody {
5634                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5635                        message: "spawn cursor belongs to a different daemon incarnation"
5636                            .to_string(),
5637                        detail: Some(serde_json::json!({
5638                            "current_daemon_incarnation": current
5639                        })),
5640                    },
5641                )?])
5642            }
5643            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5644                &frame,
5645                ErrorBody {
5646                    code: "spawn_cursor_too_old".to_string(),
5647                    message: "spawn cursor predates the retained event ring".to_string(),
5648                    detail: Some(serde_json::json!({
5649                        "oldest_retained_cursor": oldest
5650                    })),
5651                },
5652            )?]),
5653            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5654                0,
5655                frame.header.corr,
5656                format!("failed to open supervisor spawn subscription: {error}"),
5657            )),
5658        }
5659    }
5660
5661    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5662        let generation = self
5663            .registry
5664            .generation()
5665            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5666        let mut modules = Vec::new();
5667        for module in self.supervisor.list() {
5668            let status = module.status_for_control("list").map_err(|err| {
5669                RouterError::backend(
5670                    0,
5671                    frame.header.corr,
5672                    format!("failed to read supervisor status: {err}"),
5673                )
5674            })?;
5675            let (configured, _) = module.configuration().map_err(|err| {
5676                RouterError::backend(
5677                    0,
5678                    frame.header.corr,
5679                    format!("failed to read module configuration: {err}"),
5680                )
5681            })?;
5682            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5683                RouterError::backend(
5684                    0,
5685                    frame.header.corr,
5686                    format!("failed to read retry time: {err}"),
5687                )
5688            })?;
5689            // Status and configuration snapshots release their locks before the image probe awaits.
5690            let image = module.running_image_agreement().await;
5691            // Read per request so the figure is current when the operator asks;
5692            // the daemon samples nothing in between.
5693            let resources = Some(module.child_resource_usage());
5694            let mut verdict =
5695                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5696            verdict.spec = Some(launch_spec_agreement(
5697                &configured,
5698                status.process_alive,
5699                status.launch_spec_digest.as_ref(),
5700            ));
5701            let pending_reload = Some(verdict);
5702            let configured_enabled = self
5703                .capability_evaluator
5704                .configured_enabled(&status.module_id);
5705            // Keep the retired policy field on the wire for one release so
5706            // existing status consumers still receive the platform policy.
5707            modules.push(
5708                SupervisorEntry::new(
5709                    status.module_id,
5710                    status.state.to_string(),
5711                    status.enabled,
5712                    status.live,
5713                    status.health.status,
5714                )
5715                .with_configured_enabled(configured_enabled)
5716                .with_next_retry_at_ms(next_retry_at_ms)
5717                .with_launch_nonce_env(Some(!cfg!(unix)))
5718                .with_launch_nonce_source(status.launch_nonce_source)
5719                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5720                .with_protocol(status.protocol)
5721                .with_pending_reload(pending_reload)
5722                .with_last_probe_ms(status.health.last_probe_ms)
5723                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5724                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5725                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5726                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5727                .with_restart_count(Some(status.restart_count))
5728                .with_max_restarts(Some(status.max_restarts))
5729                .with_lifetime_restarts(Some(status.lifetime_restarts))
5730                .with_spawn_generation(Some(status.spawn_generation))
5731                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5732                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5733                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5734                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5735                .with_resources(resources),
5736            );
5737        }
5738        let response = ClientControlResponse::SupervisorList {
5739            generation,
5740            modules,
5741        };
5742        Ok(vec![control_response_body_frame(
5743            &frame,
5744            &response,
5745            "ClientControlResponse::SupervisorList",
5746        )?])
5747    }
5748
5749    fn handle_supervisor_stderr_tail(
5750        &self,
5751        frame: Frame,
5752        module_id: String,
5753        max_lines: Option<u32>,
5754        max_bytes: Option<u32>,
5755    ) -> Result<Vec<Frame>, RouterError> {
5756        let Some(module) = self.supervisor.get(&module_id) else {
5757            return Ok(vec![control_error_frame(
5758                &frame,
5759                "unknown_module",
5760                format!("module_id '{module_id}' is not supervised"),
5761            )?]);
5762        };
5763
5764        let snapshot = module.stderr_tail(
5765            max_lines.map(|value| value as usize),
5766            max_bytes.map(|value| value as usize),
5767        );
5768
5769        let response = ClientControlResponse::SupervisorStderrTail {
5770            module_id,
5771            tail: StderrTail {
5772                capture: match snapshot.capture {
5773                    CaptureState::Captured => StderrCaptureState::Captured,
5774                    CaptureState::Incomplete { reason } => {
5775                        StderrCaptureState::Incomplete { reason }
5776                    }
5777                    CaptureState::NotCaptured { reason } => {
5778                        StderrCaptureState::NotCaptured { reason }
5779                    }
5780                },
5781                entries: snapshot
5782                    .entries
5783                    .into_iter()
5784                    .map(|entry| match entry {
5785                        TailEntry::Line {
5786                            text,
5787                            truncated,
5788                            at_ms,
5789                        } => StderrTailEntry::Line {
5790                            text,
5791                            truncated,
5792                            at_ms,
5793                        },
5794                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5795                    })
5796                    .collect(),
5797                dropped_lines: snapshot.dropped_lines,
5798            },
5799        };
5800        Ok(vec![control_response_body_frame(
5801            &frame,
5802            &response,
5803            "ClientControlResponse::SupervisorStderrTail",
5804        )?])
5805    }
5806
5807    async fn handle_supervisor_terminals(
5808        &self,
5809        frame: Frame,
5810        module_id: String,
5811    ) -> Result<Vec<Frame>, RouterError> {
5812        let Some(module) = self.supervisor.get(&module_id) else {
5813            return Ok(vec![control_error_frame(
5814                &frame,
5815                "unknown_module",
5816                format!("module_id '{module_id}' is not supervised"),
5817            )?]);
5818        };
5819
5820        // The journal read runs on a blocking thread: it can be megabytes of
5821        // file I/O and must not occupy a runtime worker.
5822        let terminals = module
5823            .read_durable_terminal_history()
5824            .await
5825            .map_err(|error| {
5826                RouterError::backend(
5827                    0,
5828                    frame.header.corr,
5829                    format!("failed to read terminal history: {error}"),
5830                )
5831            })?;
5832        let response = ClientControlResponse::SupervisorTerminals {
5833            module_id,
5834            terminals,
5835        };
5836        Ok(vec![control_response_body_frame(
5837            &frame,
5838            &response,
5839            "ClientControlResponse::SupervisorTerminals",
5840        )?])
5841    }
5842
5843    fn handle_supervisor_routes(
5844        &self,
5845        frame: Frame,
5846        module_id: Option<String>,
5847    ) -> Result<Vec<Frame>, RouterError> {
5848        let modules = self
5849            .forwarding
5850            .route_census(module_id.as_deref())
5851            .map_err(RouterError::Forwarding)?
5852            .into_iter()
5853            .map(|(module_id, routes)| SupervisorRouteModule {
5854                module_id,
5855                routes: routes
5856                    .into_iter()
5857                    .map(|route| SupervisorRoute {
5858                        consumer: match route.principal {
5859                            Principal::Reserved { module_id } => {
5860                                SupervisorRouteConsumer::Reserved { module_id }
5861                            }
5862                            Principal::Direct | Principal::Unverified => {
5863                                SupervisorRouteConsumer::Direct {
5864                                    connection_id: route.goodbye_target.connection_id.get(),
5865                                }
5866                            }
5867                        },
5868                        age_ms: Instant::now()
5869                            .saturating_duration_since(route.bound_at)
5870                            .as_millis()
5871                            .try_into()
5872                            .unwrap_or(u64::MAX),
5873                        draining: route.draining,
5874                        drain_reason: route.drain_reason,
5875                    })
5876                    .collect(),
5877            })
5878            .collect();
5879        let response = ClientControlResponse::SupervisorRoutes { modules };
5880        Ok(vec![control_response_body_frame(
5881            &frame,
5882            &response,
5883            "ClientControlResponse::SupervisorRoutes",
5884        )?])
5885    }
5886
5887    async fn handle_supervisor_provenance(
5888        &self,
5889        frame: Frame,
5890        module_id: Option<String>,
5891    ) -> Result<Vec<Frame>, RouterError> {
5892        let mut selected = if let Some(module_id) = module_id {
5893            let Some(module) = self.supervisor.get(&module_id) else {
5894                return Ok(vec![control_error_frame(
5895                    &frame,
5896                    "unknown_module",
5897                    format!("module_id '{module_id}' is not supervised"),
5898                )?]);
5899            };
5900            vec![module]
5901        } else {
5902            self.supervisor.list()
5903        };
5904
5905        let mut modules = Vec::with_capacity(selected.len());
5906        for module in selected.drain(..) {
5907            let (status, observed_image) = module
5908                .status_and_running_image_agreement()
5909                .await
5910                .map_err(|err| {
5911                    RouterError::backend(
5912                        0,
5913                        frame.header.corr,
5914                        format!("failed to read supervisor status: {err}"),
5915                    )
5916                })?;
5917            let module_declared = self
5918                .registry
5919                .get_module(&status.module_id)
5920                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5921                .and_then(|registration| registration.manifest.provenance)
5922                .map(|build| ModuleDeclaredProvenance::Reported { build })
5923                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5924            #[cfg(test)]
5925            let running_image = match &self.provenance_probe_override {
5926                Some(result) => result.clone(),
5927                None => observed_image,
5928            };
5929            #[cfg(not(test))]
5930            let running_image = observed_image;
5931            modules.push(SupervisorModuleProvenance {
5932                module_id: status.module_id,
5933                module_declared,
5934                daemon_observed: SupervisorObservedProcess {
5935                    launch_nonce_source: status.launch_nonce_source,
5936                    pid: status.pid,
5937                    spawned_at_ms: status.spawned_at_ms,
5938                    spawned_from: status.spawned_from,
5939                    running_image,
5940                },
5941            });
5942        }
5943        let daemon = SupervisorDaemonProvenance {
5944            daemon_build: self.daemon_provenance.build.clone(),
5945            daemon_observed: DaemonObservedProcess {
5946                pid: self.daemon_provenance.pid,
5947                started_at_ms: self
5948                    .daemon_provenance
5949                    .start_clock
5950                    .map(|clock| clock.started_at_ms())
5951                    .or(self.daemon_provenance.started_at_ms),
5952                running_image: self
5953                    .daemon_provenance
5954                    .probe
5955                    .observe(
5956                        self.daemon_provenance.pid,
5957                        self.daemon_provenance.executable_path.as_deref(),
5958                        self.daemon_provenance.executable_identity.clone(),
5959                        self.daemon_provenance.process_start_time,
5960                    )
5961                    .await,
5962            },
5963        };
5964        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5965        Ok(vec![control_response_body_frame(
5966            &frame,
5967            &response,
5968            "ClientControlResponse::SupervisorProvenance",
5969        )?])
5970    }
5971
5972    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5973        self.refresh_capability_requirements();
5974        let generation = self
5975            .registry
5976            .generation()
5977            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5978        let modules = self
5979            .supervisor
5980            .list()
5981            .into_iter()
5982            .map(|module| {
5983                let status = module.status_for_control("health").map_err(|err| {
5984                    RouterError::backend(
5985                        0,
5986                        frame.header.corr,
5987                        format!("failed to read supervisor health: {err}"),
5988                    )
5989                })?;
5990                let module_id = status.module_id;
5991                let capability_detail = self
5992                    .capability_evaluator
5993                    .required_problem_detail(&module_id);
5994                Ok(SupervisorHealthEntry {
5995                    module_id,
5996                    status: status.health.status,
5997                    detail: append_capability_problem_detail(
5998                        status.health.detail,
5999                        capability_detail,
6000                    ),
6001                    metrics: status.health.metrics,
6002                    consecutive_failures: status.health.consecutive_failures,
6003                    late_answer_count: status.health.late_answer_count,
6004                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6005                    last_action: status.health.last_action,
6006                    last_action_ms: status.health.last_action_ms,
6007                    last_probe_ms: status.health.last_probe_ms,
6008                })
6009            })
6010            .collect::<Result<Vec<_>, RouterError>>()?;
6011        let response = ClientControlResponse::SupervisorHealth {
6012            generation,
6013            modules,
6014        };
6015        Ok(vec![control_response_body_frame(
6016            &frame,
6017            &response,
6018            "ClientControlResponse::SupervisorHealth",
6019        )?])
6020    }
6021
6022    async fn handle_supervisor_restart(
6023        &self,
6024        frame: Frame,
6025        module_id: String,
6026        drain_timeout_ms: Option<u64>,
6027        requester: Principal,
6028    ) -> Result<Vec<Frame>, RouterError> {
6029        let operation_lock = self.supervisor.operation_lock();
6030        let _operation_guard = operation_lock.lock().await;
6031        let Some(module) = self.supervisor.get(&module_id) else {
6032            return Ok(vec![control_error_frame(
6033                &frame,
6034                "unknown_module",
6035                format!("module_id '{module_id}' is not supervised"),
6036            )?]);
6037        };
6038
6039        self.route_outages.mark_operator_action(&module_id);
6040        if let Err(err) = module
6041            .restart_requested(drain_timeout_ms, Some(requester))
6042            .await
6043        {
6044            self.route_outages
6045                .operator_action_ended_unrefused(&module_id);
6046            let (code, message) = match err {
6047                crate::supervise::SuperviseError::Disabled { .. } => {
6048                    ("module_disabled", err.to_string())
6049                }
6050                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6051                    ("swap_in_progress", err.to_string())
6052                }
6053                _ => (
6054                    "target_unavailable",
6055                    format!("failed to restart module_id '{module_id}': {err}"),
6056                ),
6057            };
6058            return Ok(vec![control_error_frame(&frame, code, message)?]);
6059        }
6060
6061        let response = ClientControlResponse::SupervisorAck {
6062            module_id,
6063            applied: true,
6064        };
6065        Ok(vec![control_response_body_frame(
6066            &frame,
6067            &response,
6068            "ClientControlResponse::SupervisorAck",
6069        )?])
6070    }
6071
6072    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
6073    /// when the old process has finished draining: a caller whose own lane
6074    /// rides the old process must get its reply before that drain waits on it.
6075    async fn handle_supervisor_swap(
6076        &self,
6077        frame: Frame,
6078        module_id: String,
6079        ready_timeout_ms: Option<u64>,
6080        requester: Principal,
6081    ) -> Result<Vec<Frame>, RouterError> {
6082        // The daemon-wide operation lock is held only to resolve the handle,
6083        // not across the swap. The swap can take its whole readiness budget,
6084        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6085        // holding it here would park an operator's stop behind the swap it is
6086        // meant to abort. A rescan or stop that reaches the module during the
6087        // swap is served by the swap itself (see `supervise_swap`).
6088        let module = {
6089            let operation_lock = self.supervisor.operation_lock();
6090            let _operation_guard = operation_lock.lock().await;
6091            self.supervisor.get(&module_id)
6092        };
6093        let Some(module) = module else {
6094            return Ok(vec![control_error_frame(
6095                &frame,
6096                "unknown_module",
6097                format!("module_id '{module_id}' is not supervised"),
6098            )?]);
6099        };
6100
6101        self.route_outages.mark_operator_action(&module_id);
6102        if let Err(err) = module
6103            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6104            .await
6105        {
6106            self.route_outages
6107                .operator_action_ended_unrefused(&module_id);
6108            use crate::supervise::SuperviseError;
6109            let message = err.to_string();
6110            let error = match err {
6111                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6112                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6113                    code: "swap_refused".to_string(),
6114                    message,
6115                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6116                },
6117                SuperviseError::SwapFailed {
6118                    arm,
6119                    candidate_exit,
6120                    ..
6121                } => ErrorBody {
6122                    code: "swap_failed".to_string(),
6123                    message,
6124                    detail: Some(serde_json::json!({
6125                        "arm": arm.as_str(),
6126                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6127                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6128                    })),
6129                },
6130                _ => ErrorBody::new(
6131                    "target_unavailable",
6132                    format!("failed to swap module_id '{module_id}': {message}"),
6133                ),
6134            };
6135            return Ok(vec![control_error_body_frame(&frame, error)?]);
6136        }
6137        // A completed swap kept the incumbent serving until cutover, so it
6138        // usually opened no outage; a mark left behind would make the next,
6139        // unrelated outage read as requested.
6140        self.route_outages
6141            .operator_action_ended_unrefused(&module_id);
6142
6143        let response = ClientControlResponse::SupervisorAck {
6144            module_id,
6145            applied: true,
6146        };
6147        Ok(vec![control_response_body_frame(
6148            &frame,
6149            &response,
6150            "ClientControlResponse::SupervisorAck",
6151        )?])
6152    }
6153
6154    async fn handle_supervisor_reload(
6155        &self,
6156        frame: Frame,
6157        module_id: String,
6158        requester: Principal,
6159    ) -> Result<Vec<Frame>, RouterError> {
6160        let operation_lock = self.supervisor.operation_lock();
6161        let _operation_guard = operation_lock.lock().await;
6162        let Some(module) = self.supervisor.get(&module_id) else {
6163            return Ok(vec![control_error_frame(
6164                &frame,
6165                "unknown_module",
6166                format!("module_id '{module_id}' is not supervised"),
6167            )?]);
6168        };
6169
6170        self.route_outages.mark_operator_action(&module_id);
6171        if let Err(err) = module.reload_requested(Some(requester)).await {
6172            self.route_outages
6173                .operator_action_ended_unrefused(&module_id);
6174            let (code, message) = match err {
6175                crate::supervise::SuperviseError::Disabled { .. } => {
6176                    ("module_disabled", err.to_string())
6177                }
6178                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6179                    ("swap_in_progress", err.to_string())
6180                }
6181                _ => (
6182                    "reload_failed",
6183                    format!("failed to reload module_id '{module_id}': {err}"),
6184                ),
6185            };
6186            return Ok(vec![control_error_frame(&frame, code, message)?]);
6187        }
6188
6189        let response = ClientControlResponse::SupervisorAck {
6190            module_id,
6191            applied: true,
6192        };
6193        Ok(vec![control_response_body_frame(
6194            &frame,
6195            &response,
6196            "ClientControlResponse::SupervisorAck",
6197        )?])
6198    }
6199
6200    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6201    /// the configuration comes from and whether to apply it; every mode then
6202    /// runs the same comparison, so a preview describes the apply it previews.
6203    async fn handle_supervisor_rescan(
6204        &self,
6205        frame: Frame,
6206        mode: RescanMode,
6207        requester: Principal,
6208    ) -> Result<Vec<Frame>, RouterError> {
6209        let preview = mode.previews();
6210        let Some(context) = self.rescan.clone() else {
6211            return Ok(vec![control_error_frame(
6212                &frame,
6213                "rescan_unavailable",
6214                "the daemon was not started with a reloadable config path".to_string(),
6215            )?]);
6216        };
6217
6218        let operation_lock = self.supervisor.operation_lock();
6219        let _operation_guard = operation_lock.lock().await;
6220        // Loaded under the operation lock, so no other rescan can change the
6221        // staged configuration between a checked apply's comparison and the
6222        // apply itself.
6223        let loaded = match self.load_rescan_config(&mode, &context) {
6224            Ok(config) => config,
6225            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6226        };
6227        // `load` reports a missing file as Ok(None), which is correct at boot
6228        // (no config, nothing to supervise) and catastrophic here: rescan treats
6229        // "not in the config" as "remove it", so an absent file would read as an
6230        // empty module list and retire the entire running fleet. An editor
6231        // writing via write-new-then-rename, or a half-finished edit, is enough
6232        // to open that window. Refuse instead: a config that cannot be read
6233        // carries no instruction to remove anything.
6234        let Some((config, activity_config)) = loaded else {
6235            return Ok(vec![control_error_frame(
6236                &frame,
6237                "invalid_daemon_config",
6238                format!(
6239                    "daemon config not found at {}; refusing to rescan (an absent config would \
6240                     retire every supervised module)",
6241                    context.config_path.display()
6242                ),
6243            )?]);
6244        };
6245        let (
6246            configured_port,
6247            storage_config,
6248            admission_facts_carrier_module_id,
6249            admission_facts_targets,
6250            scope_authority_owners,
6251            privileged_operator_prompt,
6252            modules,
6253            reserved_capabilities,
6254        ) = (
6255            config.port,
6256            config.storage,
6257            config.admission_facts_carrier_module_id,
6258            config.admission_facts_targets,
6259            config.scope_authority_owners,
6260            config.privileged_operator_prompt,
6261            config.modules,
6262            config.reserved_capabilities,
6263        );
6264
6265        // Collect the sections rescan cannot apply, so the REPLY carries them.
6266        //
6267        // The warning below has always been correct and has always gone only to
6268        // the journal -- addressed to whoever reads logs, while the person who
6269        // just edited the config is looking at the CLI. Naming each section
6270        // individually rather than setting a flag: "something outside modules
6271        // changed" sends the operator back to diffing their own file, which is
6272        // the work this is meant to save.
6273        let mut restart_required = Vec::new();
6274        for section in RestartRequiredSection::ALL {
6275            let changed = match section {
6276                RestartRequiredSection::Port => configured_port != context.configured_port,
6277                RestartRequiredSection::Storage => storage_config != context.storage_config,
6278                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6279                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6280                }
6281                RestartRequiredSection::AdmissionFactsTargets => {
6282                    admission_facts_targets != context.admission_facts_targets
6283                }
6284                RestartRequiredSection::ScopeAuthorityOwners => {
6285                    scope_authority_owners != context.scope_authority_owners
6286                }
6287                RestartRequiredSection::PrivilegedOperatorPrompt => {
6288                    privileged_operator_prompt != context.privileged_operator_prompt
6289                }
6290            };
6291            if changed {
6292                restart_required.push(section.label().to_string());
6293            }
6294        }
6295        if !restart_required.is_empty() {
6296            warn!(
6297                config_path = %context.config_path.display(),
6298                sections = %restart_required.join(", "),
6299                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6300            );
6301        }
6302
6303        for configured in &modules {
6304            if let Err(err) = validate_spec(&configured.module_spec()) {
6305                return Ok(vec![control_error_frame(
6306                    &frame,
6307                    "invalid_daemon_config",
6308                    format!("supervisor rescan rejected daemon config: {err}"),
6309                )?]);
6310            }
6311        }
6312
6313        let configured_capabilities = modules
6314            .iter()
6315            .map(|module| (module.module_id.clone(), module.enabled))
6316            .collect::<Vec<_>>();
6317        let preview_capability_warnings = if preview {
6318            let (_, registrations) = self.runtime_capability_snapshot()?;
6319            let current_modules = self
6320                .supervisor
6321                .list()
6322                .into_iter()
6323                .map(|module| module.module_id().to_string())
6324                .collect::<BTreeSet<_>>();
6325            let resulting_modules = configured_capabilities.clone();
6326            let removed = current_modules
6327                .into_iter()
6328                .filter(|module_id| {
6329                    !resulting_modules
6330                        .iter()
6331                        .any(|(configured_id, _)| configured_id == module_id)
6332                })
6333                .collect::<Vec<_>>();
6334            // A candidate preview also names providers the candidate disables.
6335            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6336            let disabled = match &mode {
6337                RescanMode::Candidate(_) => resulting_modules
6338                    .iter()
6339                    .filter(|(_, enabled)| !enabled)
6340                    .map(|(module_id, _)| module_id.clone())
6341                    .collect::<Vec<_>>(),
6342                _ => Vec::new(),
6343            };
6344            self.capability_evaluator.preview_warnings(
6345                resulting_modules,
6346                &removed,
6347                &disabled,
6348                &registrations,
6349            )
6350        } else {
6351            Vec::new()
6352        };
6353        let reconciled = match &mode {
6354            RescanMode::Candidate(candidate) => self
6355                .preview_candidate(&context, modules, candidate)
6356                .map(|result| (result, Vec::new())),
6357            _ => {
6358                self.reconcile_supervised_modules(
6359                    &context.supervisor,
6360                    modules,
6361                    preview,
6362                    requester.clone(),
6363                )
6364                .await
6365            }
6366        };
6367        let (result, changed) = match reconciled {
6368            Ok(result) => result,
6369            Err(message) => {
6370                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6371            }
6372        };
6373        if !preview {
6374            if let Some(ledger) = &self.activity {
6375                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6376                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6377            }
6378            if let Err(code) = self.configure_activity(activity_config).await {
6379                warn!(
6380                    code,
6381                    "activity retention could not be applied during rescan"
6382                );
6383            }
6384            self.capability_evaluator
6385                .configure(configured_capabilities, reserved_capabilities);
6386            self.capability_evaluator.wake_deadline_loop();
6387            self.refresh_capability_requirements();
6388        }
6389        let mut result = result;
6390        result.restart_required = restart_required;
6391        result.capability_warnings = preview_capability_warnings;
6392        let response = ClientControlResponse::SupervisorRescan { result };
6393        Ok(vec![control_response_body_frame(
6394            &frame,
6395            &response,
6396            "ClientControlResponse::SupervisorRescan",
6397        )?])
6398    }
6399
6400    async fn handle_supervisor_release_reserved(
6401        &self,
6402        frame: Frame,
6403        module_id: String,
6404    ) -> Result<Vec<Frame>, RouterError> {
6405        let Some(context) = self.rescan.clone() else {
6406            return Ok(vec![control_error_frame(
6407                &frame,
6408                "release_unavailable",
6409                "reserved-id release requires a daemon started with a reloadable config path",
6410            )?]);
6411        };
6412        let operation_lock = self.supervisor.operation_lock();
6413        let _operation_guard = operation_lock.lock().await;
6414        let loaded = match crate::daemon_config::load(&context.config_path) {
6415            Ok(Some(config)) => config,
6416            Ok(None) => {
6417                return Ok(vec![control_error_frame(
6418                    &frame,
6419                    "invalid_daemon_config",
6420                    format!(
6421                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6422                        context.config_path.display()
6423                    ),
6424                )?])
6425            }
6426            Err(err) => {
6427                return Ok(vec![control_error_frame(
6428                    &frame,
6429                    "invalid_daemon_config",
6430                    format!("unable to verify reserved-id release against daemon config: {err}"),
6431                )?])
6432            }
6433        };
6434        if loaded
6435            .modules
6436            .iter()
6437            .any(|configured| configured.module_id == module_id)
6438        {
6439            return Ok(vec![control_error_frame(
6440                &frame,
6441                "reserved_module_configured",
6442                format!(
6443                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6444                ),
6445            )?]);
6446        }
6447        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6448            return Ok(vec![control_error_frame(
6449                &frame,
6450                "reserved_gate_not_retained",
6451                format!(
6452                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6453                ),
6454            )?]);
6455        }
6456
6457        let response = ClientControlResponse::SupervisorAck {
6458            module_id,
6459            applied: true,
6460        };
6461        Ok(vec![control_response_body_frame(
6462            &frame,
6463            &response,
6464            "ClientControlResponse::SupervisorAck",
6465        )?])
6466    }
6467
6468    /// Compare the configured module set with the supervised one, changing
6469    /// nothing. The staged spec each supervised module holds is what the
6470    /// configured entry is compared with.
6471    fn plan_rescan(
6472        &self,
6473        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6474    ) -> Result<RescanPlan, String> {
6475        let mut current = BTreeMap::new();
6476        for module in self.supervisor.list() {
6477            let (spec, health) = module.configuration().map_err(|err| {
6478                format!(
6479                    "failed to read configuration for module_id '{}': {err}",
6480                    module.module_id()
6481                )
6482            })?;
6483            let enabled = module
6484                .status()
6485                .map_err(|err| {
6486                    format!(
6487                        "failed to read status for module_id '{}': {err}",
6488                        module.module_id()
6489                    )
6490                })?
6491                .enabled;
6492            current.insert(
6493                module.module_id().to_string(),
6494                (module, spec, health, enabled),
6495            );
6496        }
6497        let configured = configured_modules
6498            .into_iter()
6499            .map(|module| (module.module_id.clone(), module))
6500            .collect::<BTreeMap<_, _>>();
6501
6502        let added = configured
6503            .keys()
6504            .filter(|module_id| !current.contains_key(*module_id))
6505            .cloned()
6506            .collect::<Vec<_>>();
6507        let removed = current
6508            .keys()
6509            .filter(|module_id| !configured.contains_key(*module_id))
6510            .cloned()
6511            .collect::<Vec<_>>();
6512        let mut changed_pending_reload = Vec::new();
6513        let mut configuration_changes = BTreeSet::new();
6514        let mut enabled_changes = BTreeSet::new();
6515        let mut unchanged = 0_u32;
6516
6517        for (module_id, configured_module) in &configured {
6518            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6519            else {
6520                continue;
6521            };
6522            // Compare the whole launch spec so a future launch field cannot
6523            // accidentally become a live-only policy change. Health is stored
6524            // separately and applies live without replacing the process.
6525            let launch_changed = *current_spec != configured_module.module_spec();
6526            let configuration_changed =
6527                launch_changed || *current_health != configured_module.health;
6528            let enabled_changed = *current_enabled != configured_module.enabled;
6529            if configuration_changed {
6530                configuration_changes.insert(module_id.clone());
6531            }
6532            if launch_changed {
6533                changed_pending_reload.push(module_id.clone());
6534            }
6535            if enabled_changed {
6536                enabled_changes.insert(module_id.clone());
6537            }
6538            if !configuration_changed && !enabled_changed {
6539                unchanged = unchanged.saturating_add(1);
6540            }
6541        }
6542        Ok(RescanPlan {
6543            current,
6544            configured,
6545            added,
6546            removed,
6547            changed_pending_reload,
6548            configuration_changes,
6549            enabled_changes,
6550            unchanged,
6551        })
6552    }
6553
6554    /// Reconcile the running module set against the configured one.
6555    ///
6556    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6557    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6558    /// deliberately shares this function with the executing path rather than
6559    /// computing the same diff somewhere else -- two implementations of one
6560    /// decision agree until they do not, and the whole value of a preview is that
6561    /// it describes the operation that will actually run.
6562    async fn reconcile_supervised_modules(
6563        &self,
6564        supervisor: &Supervisor,
6565        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6566        preview: bool,
6567        requester: Principal,
6568    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6569        let plan = self.plan_rescan(configured_modules)?;
6570
6571        // Everything above this point is pure computation over two snapshots.
6572        // Everything below MUTATES. The preview returns here so the boundary is a
6573        // single early return rather than a condition repeated at each mutation
6574        // site, where one missed guard would apply part of a change the caller was
6575        // told would not happen.
6576        if preview {
6577            return Ok((plan.preview_result(), Vec::new()));
6578        }
6579        let changed = plan
6580            .configuration_changes
6581            .union(&plan.enabled_changes)
6582            .cloned()
6583            .collect::<Vec<_>>();
6584        let RescanPlan {
6585            current,
6586            configured,
6587            added,
6588            removed,
6589            changed_pending_reload,
6590            configuration_changes,
6591            enabled_changes,
6592            unchanged,
6593        } = plan;
6594        // The staged generation moves before the first change rather than
6595        // after the last, so a rescan that fails partway still invalidates
6596        // every earlier `supervisor.rescan_checked` preview: part of its diff
6597        // may have run.
6598        if !(added.is_empty()
6599            && removed.is_empty()
6600            && configuration_changes.is_empty()
6601            && enabled_changes.is_empty())
6602        {
6603            if let Some(context) = &self.rescan {
6604                context
6605                    .generation
6606                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6607            }
6608        }
6609
6610        for module_id in &removed {
6611            let module = &current
6612                .get(module_id)
6613                .expect("removed module came from current supervisor state")
6614                .0;
6615            module
6616                .retire_requested(Some(requester.clone()))
6617                .await
6618                .map_err(|err| {
6619                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6620                })?;
6621            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6622            //
6623            // `handle_route_open` resolves an absent module in three steps:
6624            // registry, then supervisor status, then tombstone. Retiring first
6625            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6626            // went with the teardown above, the supervisor entry went with
6627            // `retire`, and the tombstone does not exist yet -- so a route.open
6628            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6629            // it") for a module that was deliberately removed and whose caller
6630            // should get `module_removed` (TERMINAL, carrying a removal age).
6631            //
6632            // Writing the tombstone first closes it: during the window the
6633            // supervisor entry still answers, so the caller gets
6634            // `target_unavailable` -- retryable, and TRUE, because the module
6635            // is mid-teardown. After both statements it is `module_removed`.
6636            // No instant remains where a removed module reads as one that
6637            // never existed.
6638            //
6639            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6640            // the absence of a test beside a fix invites deletion: these are
6641            // two sync statements with no await between them, so reaching the
6642            // window needs a second worker thread to land exactly between them
6643            // and there is no hook to force it. MEASURED: the 25 daemon_config
6644            // tests pass identically with the old order and the new one, so
6645            // the existing suite cannot see this and a green run is not
6646            // evidence either way. What the suite does hold is the
6647            // post-condition -- a removed module answers `module_removed` --
6648            // which this preserves.
6649            //
6650            // Found by an Athena panel reading the shipped tree against a
6651            // design note (2026-09-19), as the one concrete instance of that
6652            // note's class that survived contact with source. Direction is
6653            // benign: retryable where terminal was intended, never the reverse.
6654            self.supervisor.record_rescan_removal(module_id);
6655            self.supervisor.retire(module_id);
6656            self.route_outages.forget(module_id);
6657        }
6658
6659        for module_id in configured.keys() {
6660            let Some((module, _, _, _)) = current.get(module_id) else {
6661                continue;
6662            };
6663            let configured_module = configured
6664                .get(module_id)
6665                .expect("configured module id came from configured map");
6666            if configuration_changes.contains(module_id) {
6667                module
6668                    .update_configuration(
6669                        configured_module.module_spec(),
6670                        configured_module.health.clone(),
6671                        configured_module.drain_timeout_ms,
6672                    )
6673                    .await
6674                    .map_err(|err| {
6675                        format!(
6676                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6677                        )
6678                    })?;
6679            }
6680            if enabled_changes.contains(module_id) {
6681                // A rescan that starts or stops a module applies an operator's
6682                // edit to the config, so the resulting outage was asked for.
6683                self.route_outages.mark_operator_action(module_id);
6684                module
6685                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6686                    .await
6687                    .map_err(|err| {
6688                        self.route_outages.operator_action_ended_unrefused(module_id);
6689                        format!(
6690                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6691                            configured_module.enabled
6692                        )
6693                    })?;
6694            }
6695        }
6696
6697        for module_id in &added {
6698            let configured_module = configured
6699                .get(module_id)
6700                .expect("added module id came from configured map");
6701            supervisor
6702                .supervise_configured_requested(
6703                    configured_module.module_spec(),
6704                    configured_module.enabled,
6705                    configured_module.health.clone(),
6706                    configured_module.drain_timeout_ms,
6707                    configured_module.restart,
6708                    configured_module.register_timeout,
6709                    Some(requester.clone()),
6710                )
6711                .map_err(|err| {
6712                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6713                })?;
6714        }
6715
6716        Ok((
6717            SupervisorRescanResult {
6718                added,
6719                removed,
6720                changed_pending_reload,
6721                enabled_changes: enabled_changes.iter().cloned().collect(),
6722                unchanged,
6723                preview: false,
6724                // Filled by the caller, which is the only layer that can see the
6725                // previous config to diff against.
6726                restart_required: Vec::new(),
6727                capability_warnings: Vec::new(),
6728                module_actions: Vec::new(),
6729                earlier_pending: Vec::new(),
6730                expect: None,
6731            },
6732            changed,
6733        ))
6734    }
6735
6736    async fn handle_supervisor_set_enabled(
6737        &self,
6738        frame: Frame,
6739        module_id: String,
6740        enabled: bool,
6741        requester: Principal,
6742    ) -> Result<Vec<Frame>, RouterError> {
6743        let operation_lock = self.supervisor.operation_lock();
6744        let _operation_guard = operation_lock.lock().await;
6745        let Some(module) = self.supervisor.get(&module_id) else {
6746            return Ok(vec![control_error_frame(
6747                &frame,
6748                "unknown_module",
6749                format!("module_id '{module_id}' is not supervised"),
6750            )?]);
6751        };
6752
6753        // Enabling counts as well as disabling: a module an operator starts
6754        // is refused until it registers, and that wait was asked for.
6755        self.route_outages.mark_operator_action(&module_id);
6756        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6757            Ok(applied) => applied,
6758            Err(err) => {
6759                self.route_outages
6760                    .operator_action_ended_unrefused(&module_id);
6761                return Ok(vec![control_error_frame(
6762                    &frame,
6763                    "target_unavailable",
6764                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6765                )?]);
6766            }
6767        };
6768        if !applied {
6769            // Already in the requested state: nothing was made unavailable,
6770            // so the mark must not outlive this request.
6771            self.route_outages
6772                .operator_action_ended_unrefused(&module_id);
6773        }
6774
6775        self.capability_evaluator.wake_deadline_loop();
6776        self.refresh_capability_requirements();
6777        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6778        Ok(vec![control_response_body_frame(
6779            &frame,
6780            &response,
6781            "ClientControlResponse::SupervisorAck",
6782        )?])
6783    }
6784
6785    async fn handle_supervisor_health_probe(
6786        &self,
6787        frame: Frame,
6788        module_id: String,
6789    ) -> Result<Vec<Frame>, RouterError> {
6790        self.refresh_capability_requirements();
6791        let Some(registration) = self
6792            .registry
6793            .get_module(&module_id)
6794            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6795        else {
6796            return Ok(vec![control_error_frame(
6797                &frame,
6798                "unknown_module",
6799                format!("module_id '{module_id}' is not registered"),
6800            )?]);
6801        };
6802
6803        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6804        // named for it. Making `module_registration_grants_op` return false
6805        // unconditionally reddens five tests, and every one is named for something
6806        // else -- capability relay, probe/bind demultiplexing, supervision-only
6807        // probing. They exercise a successful advertisement check on the way to their
6808        // own subject.
6809        //
6810        // Real protection, fragile in a specific way: narrowing any of those tests to
6811        // focus on its stated subject would silently remove coverage nobody knows
6812        // they are carrying. Recorded here rather than as a sixth test, because the
6813        // useful fact is WHICH tests hold the guard up -- a new test would add
6814        // coverage without telling the next person what the existing ones quietly do.
6815        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6816        {
6817            return Ok(vec![control_error_frame(
6818                &frame,
6819                "health_not_advertised",
6820                format!("module_id '{module_id}' did not advertise health.check"),
6821            )?]);
6822        }
6823
6824        let deadline = Instant::now() + self.health_probe_timeout;
6825        let pending = match self.forwarding.begin_module_control_rpc_for(
6826            &module_id,
6827            MODULE_CONTROL_OP_HEALTH_CHECK,
6828            deadline,
6829        ) {
6830            Ok(pending) => pending,
6831            Err(err) => {
6832                return Ok(vec![control_error_frame(
6833                    &frame,
6834                    forwarding_error_code(&err),
6835                    err.to_string(),
6836                )?])
6837            }
6838        };
6839
6840        let PendingModuleControlRpc {
6841            endpoint,
6842            module_sink,
6843            negotiated_ver,
6844            corr: probe_corr,
6845            receiver,
6846        } = pending;
6847        let mut guard =
6848            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6849        let probe_body =
6850            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6851                RouterError::backend(
6852                    0,
6853                    frame.header.corr,
6854                    format!("failed to encode health.check request: {err}"),
6855                )
6856            })?;
6857        let probe_frame = Frame::build_with_version(
6858            negotiated_ver,
6859            FrameType::Request,
6860            control_flags(),
6861            0,
6862            0,
6863            probe_corr,
6864            probe_body,
6865        )
6866        .map_err(RouterError::FrameBuild)?;
6867
6868        // Queue admission and the reply share one deadline, so a module that
6869        // stops reading cannot park the requesting connection's reader forever.
6870        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6871            Ok(Ok(())) => {}
6872            Ok(Err(err)) => {
6873                return Ok(vec![control_error_frame(
6874                    &frame,
6875                    "target_unavailable",
6876                    err.to_string(),
6877                )?]);
6878            }
6879            Err(_) => {
6880                return Ok(vec![control_error_frame(
6881                    &frame,
6882                    "module_timeout",
6883                    format!(
6884                        "module_id '{module_id}' did not answer health.check within {:?}",
6885                        self.health_probe_timeout
6886                    ),
6887                )?]);
6888            }
6889        }
6890
6891        match timeout_at(deadline, receiver).await {
6892            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6893                guard.disarm();
6894                let Some(report) = response.health_report() else {
6895                    return Ok(vec![control_error_frame(
6896                        &frame,
6897                        "invalid_control_body",
6898                        "health.check RPC returned a non-health response",
6899                    )?]);
6900                };
6901                // Metrics go out whole here. The supervisor's cached snapshot
6902                // caps this blob (see truncate_health_metrics), and this path
6903                // exists precisely to answer without that cap -- so applying it
6904                // here would leave no way to see what the cached view drops.
6905                let HealthReport {
6906                    status,
6907                    detail,
6908                    metrics,
6909                } = report;
6910                let capability_detail = self
6911                    .capability_evaluator
6912                    .required_problem_detail(&module_id);
6913                let response = ClientControlResponse::SupervisorHealthProbe {
6914                    module_id,
6915                    status,
6916                    detail: append_capability_problem_detail(detail, capability_detail),
6917                    metrics,
6918                };
6919                Ok(vec![control_response_body_frame(
6920                    &frame,
6921                    &response,
6922                    "ClientControlResponse::SupervisorHealthProbe",
6923                )?])
6924            }
6925            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6926                guard.disarm();
6927                // The module refused the health probe with its own body, so
6928                // the refusal is relayed without the daemon-origin bit.
6929                Ok(vec![control_relayed_error_body_frame(&frame, body)?])
6930            }
6931            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6932                guard.disarm();
6933                Ok(vec![control_error_frame(
6934                    &frame,
6935                    "target_unavailable",
6936                    message,
6937                )?])
6938            }
6939            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6940                guard.disarm();
6941                Ok(vec![control_error_frame(
6942                    &frame,
6943                    "invalid_control_body",
6944                    message,
6945                )?])
6946            }
6947            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6948                guard.disarm();
6949                Ok(vec![control_error_frame(
6950                    &frame,
6951                    "invalid_control_body",
6952                    format!("expected module-control op '{expected}', got '{actual}'"),
6953                )?])
6954            }
6955            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6956                guard.disarm();
6957                Ok(vec![control_error_frame(
6958                    &frame,
6959                    "module_timeout",
6960                    format!(
6961                        "module_id '{module_id}' answered health.check after {:?}",
6962                        self.health_probe_timeout
6963                    ),
6964                )?])
6965            }
6966            Ok(Err(_)) => Ok(vec![control_error_frame(
6967                &frame,
6968                "target_unavailable",
6969                "health.check waiter was canceled before the module responded",
6970            )?]),
6971            Err(_) => Ok(vec![control_error_frame(
6972                &frame,
6973                "module_timeout",
6974                format!(
6975                    "module_id '{module_id}' did not answer health.check within {:?}",
6976                    self.health_probe_timeout
6977                ),
6978            )?]),
6979        }
6980    }
6981
6982    fn supervisor_status(
6983        &self,
6984        module_id: &str,
6985        corr: u64,
6986    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6987        self.supervisor
6988            .get(module_id)
6989            .map(|module| {
6990                let warming = module.is_warming_for_control("status").map_err(|err| {
6991                    RouterError::backend(
6992                        0,
6993                        corr,
6994                        format!(
6995                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6996                        ),
6997                    )
6998                })?;
6999                module.status_for_control("status").map_err(|err| {
7000                    RouterError::backend(
7001                        0,
7002                        corr,
7003                        format!(
7004                            "failed to read supervisor status for module_id '{module_id}': {err}"
7005                        ),
7006                    )
7007                }).map(|status| (status, warming))
7008            })
7009            .transpose()
7010    }
7011
7012    fn guard_module_control_op(
7013        &self,
7014        frame: &Frame,
7015        module_id: &str,
7016        op: &str,
7017    ) -> Result<Option<Frame>, RouterError> {
7018        if self.module_grants_op(module_id, op, frame.header.corr)? {
7019            return Ok(None);
7020        }
7021
7022        Ok(Some(control_error_frame(
7023            frame,
7024            "op_not_allowed",
7025            format!("module_id '{module_id}' did not grant control op '{op}'"),
7026        )?))
7027    }
7028
7029    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7030        let Some(registration) = self
7031            .registry
7032            .get_module(module_id)
7033            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7034        else {
7035            return Ok(false);
7036        };
7037        Ok(module_registration_grants_op(&registration.control_ops, op))
7038    }
7039
7040    fn handle_status_update(
7041        &self,
7042        endpoint: ModuleEndpointId,
7043        frame: Frame,
7044    ) -> Result<Vec<Frame>, RouterError> {
7045        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7046            Ok(update) => update,
7047            Err(err) => {
7048                // Forward-compat: a newer module may push a channel-0 op this subc
7049                // version doesn't know. The control contract says unknown push ops
7050                // are IGNORED, never answered with an error. Only a malformed body
7051                // for an op we DO know is a real error worth surfacing.
7052                if is_known_module_push_op(&frame.body) {
7053                    return Ok(vec![control_error_frame(
7054                        &frame,
7055                        "invalid_control_body",
7056                        format!("malformed module control push body: {err}"),
7057                    )?]);
7058                }
7059                return Ok(Vec::new());
7060            }
7061        };
7062
7063        match update {
7064            ModuleControlPush::RouteStatus {
7065                route_channel,
7066                route_epoch,
7067                status,
7068            } => {
7069                self.forwarding
7070                    .cache_status(endpoint, route_channel, route_epoch, status)
7071                    .map_err(RouterError::Forwarding)?;
7072            }
7073        }
7074        Ok(Vec::new())
7075    }
7076
7077    fn handle_route_poll(
7078        &self,
7079        ctx: &RouteCtx,
7080        frame: Frame,
7081        route_channel: u16,
7082        route_epoch: u32,
7083        kind: PollKind,
7084    ) -> Result<Vec<Frame>, RouterError> {
7085        let snapshot = self
7086            .forwarding
7087            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7088            .map_err(RouterError::Forwarding)?;
7089        let response = match (kind, snapshot) {
7090            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7091                ClientControlResponse::RoutePoll {
7092                    route_channel,
7093                    route_epoch,
7094                    status,
7095                    live: None,
7096                }
7097            }
7098            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7099                route_channel,
7100                route_epoch,
7101                status: None,
7102                live: None,
7103            },
7104            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7105                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7106                // is what makes reporting `true` correct rather than a
7107                // confident guess. `process_live` returns None only when the
7108                // module id has no supervisor snapshot at all -- an
7109                // externally-started module the daemon did not spawn -- and
7110                // for those the supervisor has no opinion to offer, ever. It
7111                // is never None for a supervised module in an unknown state:
7112                // a supervised module always has a snapshot, and the answer
7113                // comes from `state == Running && process_alive`.
7114                //
7115                // The route is Bound, so the module completed a HELLO on a
7116                // live connection; "the process this route points at is
7117                // running" is therefore attested by the binding rather than
7118                // assumed. Reporting `false` for an unsupervised module would
7119                // be the actual lie -- it would tell a client its healthy
7120                // route is dead because the daemon does not manage the
7121                // process.
7122                //
7123                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7124                // module whose liveness is genuinely unknown, e.g. a snapshot
7125                // that has not been populated yet -- THIS DEFAULT BECOMES
7126                // WRONG and must split: unsupervised stays true, unknown
7127                // becomes null so the client can tell the two apart. The
7128                // response field is already `Option<bool>`, so the wire can
7129                // carry that distinction today.
7130                let live = self
7131                    .process_liveness
7132                    .as_ref()
7133                    .and_then(|source| source.process_live(&module_id))
7134                    .unwrap_or(true);
7135                ClientControlResponse::RoutePoll {
7136                    route_channel,
7137                    route_epoch,
7138                    status: None,
7139                    live: Some(live),
7140                }
7141            }
7142            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7143                route_channel,
7144                route_epoch,
7145                status: None,
7146                live: Some(false),
7147            },
7148        };
7149
7150        Ok(vec![control_response_body_frame(
7151            &frame,
7152            &response,
7153            "ClientControlResponse::RoutePoll",
7154        )?])
7155    }
7156
7157    pub(crate) fn observe_module_control_completion(
7158        &self,
7159        completion: ModuleControlRpcCompletion,
7160    ) -> bool {
7161        match completion {
7162            ModuleControlRpcCompletion::Unknown => false,
7163            ModuleControlRpcCompletion::Settled => true,
7164            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7165                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7166                info!(
7167                    module_id = %module_id,
7168                    latency_ms,
7169                    "late health.check answer proves the module is alive"
7170                );
7171                match self
7172                    .supervisor
7173                    .record_late_health_answer(&module_id, latency_ms)
7174                {
7175                    Ok(true) => {}
7176                    Ok(false) => debug!(
7177                        module_id = %module_id,
7178                        latency_ms,
7179                        "late health.check answer has no active supervisor snapshot"
7180                    ),
7181                    Err(err) => warn!(
7182                        module_id = %module_id,
7183                        latency_ms,
7184                        error = %err,
7185                        "failed to record late health.check answer"
7186                    ),
7187                }
7188                true
7189            }
7190        }
7191    }
7192
7193    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7194    /// the module connection whose frame is being handled, or to the client that
7195    /// relay was opened for.
7196    ///
7197    /// This runs on the MODULE connection's frame handler, where returning `Err`
7198    /// ends that connection -- and a module connection carries every client's
7199    /// routes to that module, so ending it costs the whole fleet its tools.
7200    /// `ConnectionClosing` carries the id of the connection that is closing, and
7201    /// when that id is a CLIENT's, the condition is entirely about that one
7202    /// client's route.open. A client-scoped condition has no authority over a
7203    /// shared module connection, so it is logged and the single relay is dropped:
7204    /// the client is going away, and `complete_pending_relay` already removed the
7205    /// relay before failing, so there is nothing left to settle. Anything that
7206    /// relay still reserved is released by that client's own connection teardown,
7207    /// which is already under way -- that is what "closing" means.
7208    ///
7209    /// Every other failure is a statement about THIS connection and stays fatal:
7210    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7211    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7212    /// frames correctly.
7213    fn refuse_to_end_module_connection_for_a_client(
7214        &self,
7215        module_connection_id: ConnectionId,
7216        corr: u64,
7217        err: ForwardingError,
7218    ) -> Result<(), RouterError> {
7219        if let ForwardingError::ConnectionClosing { connection_id } = err {
7220            if connection_id != module_connection_id {
7221                warn!(
7222                    module_connection_id = module_connection_id.get(),
7223                    client_connection_id = connection_id.get(),
7224                    corr,
7225                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7226                );
7227                return Ok(());
7228            }
7229        }
7230        Err(RouterError::Forwarding(err))
7231    }
7232
7233    fn handle_module_relay_response(
7234        &self,
7235        connection_id: ConnectionId,
7236        frame: Frame,
7237    ) -> Result<Vec<Frame>, RouterError> {
7238        let mut secondary_error = None;
7239        let outcome = match frame.header.ty {
7240            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7241                Ok(probe) if probe.op == "route.bind" => {
7242                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7243                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7244                            RouteBindRelayOutcome::Accepted
7245                        }
7246                        Ok(other) => {
7247                            let message =
7248                                format!("route.bind response carried unexpected body: {other:?}");
7249                            secondary_error = Some(control_error_frame(
7250                                &frame,
7251                                "invalid_control_body",
7252                                message.clone(),
7253                            )?);
7254                            RouteBindRelayOutcome::ModuleGone(message)
7255                        }
7256                        Err(err) => {
7257                            let message = format!("malformed route.bind response body: {err}");
7258                            secondary_error = Some(control_error_frame(
7259                                &frame,
7260                                "invalid_control_body",
7261                                message.clone(),
7262                            )?);
7263                            RouteBindRelayOutcome::ModuleGone(message)
7264                        }
7265                    }
7266                }
7267                Ok(probe) => {
7268                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7269                    {
7270                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7271                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7272                            "malformed {} response body: {err}",
7273                            probe.op
7274                        )),
7275                    };
7276                    let completion = self
7277                        .forwarding
7278                        .complete_module_control_rpc(
7279                            connection_id,
7280                            frame.header.corr,
7281                            Some(&probe.op),
7282                            outcome,
7283                        )
7284                        .map_err(RouterError::Forwarding)?;
7285                    if !self.observe_module_control_completion(completion) {
7286                        debug!(
7287                            connection_id = connection_id.get(),
7288                            corr = frame.header.corr,
7289                            op = %probe.op,
7290                            "dropping late or unknown module-control RPC response"
7291                        );
7292                    }
7293                    return Ok(Vec::new());
7294                }
7295                Err(err) => {
7296                    if let Some(expected_op) = self
7297                        .forwarding
7298                        .pending_module_control_op(connection_id, frame.header.corr)
7299                        .map_err(RouterError::Forwarding)?
7300                    {
7301                        let completion = self
7302                            .forwarding
7303                            .complete_module_control_rpc(
7304                                connection_id,
7305                                frame.header.corr,
7306                                None,
7307                                ModuleControlRpcOutcome::MalformedResponse(format!(
7308                                    "malformed {expected_op} response body: {err}"
7309                                )),
7310                            )
7311                            .map_err(RouterError::Forwarding)?;
7312                        if !self.observe_module_control_completion(completion) {
7313                            debug!(
7314                                connection_id = connection_id.get(),
7315                                corr = frame.header.corr,
7316                                "dropping late malformed module-control RPC response"
7317                            );
7318                        }
7319                        return Ok(Vec::new());
7320                    }
7321                    let message = format!("malformed route.bind response body: {err}");
7322                    secondary_error = Some(control_error_frame(
7323                        &frame,
7324                        "invalid_control_body",
7325                        message.clone(),
7326                    )?);
7327                    RouteBindRelayOutcome::ModuleGone(message)
7328                }
7329            },
7330            FrameType::Error => {
7331                if self
7332                    .forwarding
7333                    .pending_module_control_op(connection_id, frame.header.corr)
7334                    .map_err(RouterError::Forwarding)?
7335                    .is_some()
7336                {
7337                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7338                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7339                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7340                            "malformed module-control ERROR body: {err}"
7341                        )),
7342                    };
7343                    let completion = self
7344                        .forwarding
7345                        .complete_module_control_rpc(
7346                            connection_id,
7347                            frame.header.corr,
7348                            None,
7349                            outcome,
7350                        )
7351                        .map_err(RouterError::Forwarding)?;
7352                    if !self.observe_module_control_completion(completion) {
7353                        debug!(
7354                            connection_id = connection_id.get(),
7355                            corr = frame.header.corr,
7356                            "dropping late or unknown module-control RPC error"
7357                        );
7358                    }
7359                    return Ok(Vec::new());
7360                }
7361                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7362                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7363                    Err(err) => {
7364                        let message = format!("malformed route.bind ERROR body: {err}");
7365                        secondary_error = Some(control_error_frame(
7366                            &frame,
7367                            "invalid_control_body",
7368                            message.clone(),
7369                        )?);
7370                        RouteBindRelayOutcome::ModuleGone(message)
7371                    }
7372                }
7373            }
7374            ty => {
7375                return Ok(vec![control_error_frame(
7376                    &frame,
7377                    "unsupported_control_frame",
7378                    format!("unsupported module channel-0 frame {ty:?}"),
7379                )?])
7380            }
7381        };
7382
7383        let settled =
7384            self.forwarding
7385                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7386        let completion = match settled {
7387            Ok(completion) => completion,
7388            Err(err) => {
7389                self.refuse_to_end_module_connection_for_a_client(
7390                    connection_id,
7391                    frame.header.corr,
7392                    err,
7393                )?;
7394                return Ok(secondary_error.into_iter().collect());
7395            }
7396        };
7397        if let Some(target) = completion.abandoned.as_ref() {
7398            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7399        }
7400        if !completion.settled {
7401            debug!(
7402                connection_id = connection_id.get(),
7403                corr = frame.header.corr,
7404                frame_type = ?frame.header.ty,
7405                "dropping late or unknown route.bind relay response"
7406            );
7407        }
7408        Ok(secondary_error.into_iter().collect())
7409    }
7410
7411    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7412        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7413        // GOODBYE ends the connection's logical session even when its socket
7414        // stays open. Use disconnect teardown so verdicts, client notices and
7415        // scope authority are released at the same lifecycle boundary.
7416        self.cleanup_connection_with_end_reason(
7417            connection_id,
7418            RegistrationEndReason::ExplicitGoodbye,
7419        )
7420        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7421        Ok(Vec::new())
7422    }
7423}
7424
7425impl Default for ControlHandler {
7426    fn default() -> Self {
7427        Self::new(Arc::new(Registry::default()))
7428    }
7429}
7430
7431impl crate::supervise::SwapPromotionObserver for ControlHandler {
7432    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7433        self.apply_registration_capabilities(registration);
7434    }
7435}
7436
7437fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7438    CapabilityRequirementStatus {
7439        consumer: status.consumer,
7440        capability: status.capability,
7441        need: match status.need {
7442            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7443            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7444        },
7445        verdict: status.verdict.as_str().to_string(),
7446        episode_seq: status.episode_seq,
7447        config_satisfiable: status.config_satisfiable,
7448        runtime_available: status.runtime_available,
7449        detail: status.detail,
7450    }
7451}
7452
7453fn append_capability_problem_detail(
7454    detail: Option<String>,
7455    capability_detail: Option<String>,
7456) -> Option<String> {
7457    match (detail, capability_detail) {
7458        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7459        (Some(detail), None) => Some(detail),
7460        (None, Some(capability_detail)) => Some(capability_detail),
7461        (None, None) => None,
7462    }
7463}
7464
7465fn subc_ops() -> Vec<String> {
7466    SUBC_CONTROL_OPS
7467        .iter()
7468        .map(|op| (*op).to_string())
7469        .collect()
7470}
7471
7472fn module_subc_ops() -> Vec<String> {
7473    SUBC_CONTROL_OPS
7474        .iter()
7475        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7476        .map(|op| (*op).to_string())
7477        .collect()
7478}
7479
7480#[cfg(test)]
7481fn module_baseline_control_ops() -> Vec<String> {
7482    MODULE_BASELINE_CONTROL_OPS
7483        .iter()
7484        .map(|op| (*op).to_string())
7485        .collect()
7486}
7487
7488fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7489    let mut seen = HashSet::new();
7490    let mut effective = Vec::new();
7491    for op in MODULE_BASELINE_CONTROL_OPS {
7492        if seen.insert((*op).to_string()) {
7493            effective.push((*op).to_string());
7494        }
7495    }
7496    for op in declared.unwrap_or_default() {
7497        if seen.insert(op.clone()) {
7498            effective.push(op);
7499        }
7500    }
7501    effective
7502}
7503
7504fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7505    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7506}
7507
7508fn target_module_id(target: &RouteTarget) -> &str {
7509    match target {
7510        RouteTarget::ToolProvider { module_id }
7511        | RouteTarget::ManagementSurface { module_id }
7512        | RouteTarget::InternalService { module_id, .. } => module_id,
7513    }
7514}
7515
7516fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7517    roles.iter().any(|role| match (target, role) {
7518        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7519        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7520        (
7521            RouteTarget::InternalService { service_id, .. },
7522            ProviderRole::InternalService {
7523                service_id: provided,
7524                ..
7525            },
7526        ) => service_id == provided,
7527        _ => false,
7528    })
7529}
7530
7531fn is_routable_role(role: &ProviderRole) -> bool {
7532    matches!(
7533        role,
7534        ProviderRole::ToolProvider { .. }
7535            | ProviderRole::ManagementSurface { .. }
7536            | ProviderRole::InternalService { .. }
7537    )
7538}
7539
7540#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7541enum ControlRequestBodyError {
7542    UnknownOp,
7543    InvalidBody,
7544}
7545
7546#[derive(Debug, Deserialize)]
7547struct ControlOpProbe {
7548    op: String,
7549}
7550
7551/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7552/// this set is treated as a forward-compat unknown and ignored rather than errored.
7553const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7554
7555fn is_known_module_push_op(body: &[u8]) -> bool {
7556    serde_json::from_slice::<ControlOpProbe>(body)
7557        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7558        .unwrap_or(false)
7559}
7560
7561fn is_known_module_request_op(body: &[u8]) -> bool {
7562    serde_json::from_slice::<ControlOpProbe>(body)
7563        .map(|probe| is_module_to_subc_op(&probe.op))
7564        .unwrap_or(false)
7565}
7566
7567fn is_module_to_subc_op(op: &str) -> bool {
7568    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7569}
7570
7571fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7572    debug!(
7573        op = %op,
7574        connection_id = connection_id.get(),
7575        corr,
7576        "control dispatch"
7577    );
7578}
7579
7580fn log_slow_control_dispatch(
7581    dispatch_started_at: Option<StdInstant>,
7582    op: &'static str,
7583    connection_id: ConnectionId,
7584    corr: u64,
7585    route_open_timings: Option<&RouteOpenTimings>,
7586) {
7587    let Some(dispatch_started_at) = dispatch_started_at else {
7588        return;
7589    };
7590    // Use the same monotonic clock as the phase timers and relay deadline.
7591    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7592    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7593        if let Some(timings) = route_open_timings {
7594            warn!(
7595                op = %op,
7596                connection_id = connection_id.get(),
7597                corr,
7598                elapsed_ms = elapsed.as_millis() as u64,
7599                module_id = ?timings.module_id,
7600                principal = %timings.principal,
7601                admission_ms = timings.admission.as_millis() as u64,
7602                project_root_ms = timings.project_root.as_millis() as u64,
7603                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7604                bind_send_ms = timings.bind_send.as_millis() as u64,
7605                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7606                "slow control dispatch"
7607            );
7608            return;
7609        }
7610        warn!(
7611            op = %op,
7612            connection_id = connection_id.get(),
7613            corr,
7614            elapsed_ms = elapsed.as_millis() as u64,
7615            "slow control dispatch"
7616        );
7617    }
7618}
7619
7620/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7621#[cfg(test)]
7622type TestCallerLookup =
7623    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7624
7625/// Every new client operation needs an explicit privilege decision here.
7626fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7627    match request {
7628        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7629            Some(PrivilegedOperation::Restart { module_id })
7630        }
7631        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7632            Some(PrivilegedOperation::Swap { module_id })
7633        }
7634        ClientControlRequest::SupervisorReload { module_id } => {
7635            Some(PrivilegedOperation::Reload { module_id })
7636        }
7637        ClientControlRequest::SupervisorRescan { preview } => {
7638            if *preview {
7639                None
7640            } else {
7641                Some(PrivilegedOperation::Rescan)
7642            }
7643        }
7644        // Only the well-formed apply asks for approval. A preview changes
7645        // nothing, and any other combination is refused before it touches
7646        // anything, so prompting the operator for it would ask about a request
7647        // that cannot run.
7648        ClientControlRequest::SupervisorRescanChecked {
7649            preview,
7650            candidate,
7651            expect,
7652        } => (!*preview && candidate.is_none() && expect.is_some())
7653            .then_some(PrivilegedOperation::Rescan),
7654        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7655            Some(PrivilegedOperation::SetEnabled {
7656                module_id,
7657                enabled: *enabled,
7658            })
7659        }
7660        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7661            Some(PrivilegedOperation::ReleaseReserved { module_id })
7662        }
7663        ClientControlRequest::ServerDescribe {}
7664        | ClientControlRequest::CatalogList { .. }
7665        | ClientControlRequest::RouteOpen { .. }
7666        | ClientControlRequest::RoutePoll { .. }
7667        | ClientControlRequest::SupervisorList {}
7668        | ClientControlRequest::SupervisorHealth {}
7669        | ClientControlRequest::SupervisorHealthProbe { .. }
7670        | ClientControlRequest::SupervisorRoutes { .. }
7671        | ClientControlRequest::SupervisorProvenance { .. }
7672        | ClientControlRequest::SupervisorStderrTail { .. }
7673        | ClientControlRequest::SupervisorTerminals { .. }
7674        | ClientControlRequest::SupervisorSpawnSnapshot {}
7675        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7676    }
7677}
7678
7679fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7680    match request {
7681        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7682        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7683        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7684        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7685        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7686        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7687        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7688        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7689        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7690        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7691        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7692        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7693        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7694        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7695        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7696        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7697        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7698        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7699        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7700        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7701    }
7702}
7703
7704/// The caller of one `route.open`, as its `route.open refused` line names it.
7705///
7706/// A connection id alone cannot name a caller after the fact: ids restart at
7707/// every daemon restart, so the same number belongs to different modules
7708/// before and after one. These three fields are what the refused line needs
7709/// to be attributable on its own, and they are rendered with the same helpers
7710/// as the `route.open accepted` line so both directions grep alike.
7711struct RouteOpenCaller {
7712    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
7713    /// when the request presented a consumer identity whose launch nonce did
7714    /// not verify. The claimed module id is never printed as a principal.
7715    principal: String,
7716    /// `none` for an unscoped open; the admitted stamp's label once scope
7717    /// admission issued one; before that, the requested selector's label.
7718    scope: String,
7719    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
7720    /// because it is a string the caller chose.
7721    harness: String,
7722}
7723
7724impl RouteOpenCaller {
7725    fn new(
7726        principal: Option<&Principal>,
7727        scope: Option<&ScopeSelector>,
7728        identity: &BindIdentity,
7729    ) -> Self {
7730        Self {
7731            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7732            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7733            harness: bounded_log_text(&identity.harness),
7734        }
7735    }
7736
7737    /// For a refusal that has no parsed request at hand: names whatever the
7738    /// frame still says, and `unknown` where it says nothing parseable.
7739    fn unknown() -> Self {
7740        Self {
7741            principal: "unknown".to_string(),
7742            scope: "unknown".to_string(),
7743            harness: "unknown".to_string(),
7744        }
7745    }
7746}
7747
7748/// Longest client-chosen text (a scope ref, a harness) a route.open log line
7749/// prints before cutting it.
7750const MAX_LOGGED_TEXT_CHARS: usize = 64;
7751
7752/// `value` Debug-quoted, so control characters land escaped and cannot forge
7753/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
7754/// `…` inside the quotes so one request cannot flood the line.
7755fn bounded_log_text(value: &str) -> String {
7756    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7757    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7758        bounded.push('…');
7759    }
7760    format!("{bounded:?}")
7761}
7762
7763/// The `scope=` field of the `route.open accepted` line, from the stamp that
7764/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7765/// so a ref holding control characters cannot forge a log line, and cut to 64
7766/// characters with a trailing `…` to bound the line. The bind proof and the
7767/// scope's attributes never appear.
7768fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7769    use subc_protocol::scope::ScopeKind;
7770    let kind = match stamp.kind {
7771        ScopeKind::Head => "head",
7772        ScopeKind::Worker => "worker",
7773        ScopeKind::Ephemeral => "ephemeral",
7774    };
7775    format!(
7776        "{kind}/{}/{}@{}",
7777        principal_label(&stamp.owner),
7778        bounded_log_text(&stamp.scope_ref),
7779        stamp.scope_epoch
7780    )
7781}
7782
7783/// The `scope=` field of a `route.open refused` line for an open refused
7784/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
7785/// The kind is not known yet, so the literal `requested` stands in its place.
7786/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
7787/// owner it has not been checked against the configured modules: a quoted
7788/// owner means the value was never validated. The bind proof never appears.
7789fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7790    let epoch = selector
7791        .scope_epoch
7792        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7793    format!(
7794        "requested/{}/{}@{epoch}",
7795        bounded_log_text(&principal_label(&selector.owner)),
7796        bounded_log_text(&selector.scope_ref),
7797    )
7798}
7799
7800fn principal_label(principal: &Principal) -> String {
7801    match principal {
7802        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7803        Principal::Direct => "direct".to_string(),
7804        other => format!("{other:?}"),
7805    }
7806}
7807
7808fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7809    match request {
7810        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7811        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7812        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7813        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7814        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7815    }
7816}
7817
7818fn parse_client_control_request(
7819    body: &[u8],
7820) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7821    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7822        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7823            Ok(probe)
7824                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7825                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7826            {
7827                ControlRequestBodyError::InvalidBody
7828            }
7829            Ok(_) => ControlRequestBodyError::UnknownOp,
7830            Err(_) => ControlRequestBodyError::InvalidBody,
7831        };
7832        (err, classification)
7833    })
7834}
7835
7836fn parse_module_control_request_from_module(
7837    body: &[u8],
7838) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7839    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7840        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7841            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7842            Ok(_) => ControlRequestBodyError::UnknownOp,
7843            Err(_) => ControlRequestBodyError::InvalidBody,
7844        };
7845        (err, classification)
7846    })
7847}
7848
7849#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7850enum ProviderRoleKind {
7851    ToolProvider,
7852    PipelineStage,
7853    ManagementSurface,
7854    InternalService,
7855}
7856
7857fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7858    match role {
7859        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7860        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7861        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7862        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7863    }
7864}
7865
7866fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7867    roles.iter().map(provider_role_kind).collect()
7868}
7869
7870/// Most refused scope records named individually in the log per sync; the
7871/// `refused` count on the accepted line is always complete.
7872const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7873
7874/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7875#[derive(Debug, Default, PartialEq, Eq)]
7876struct ScopeOutcomeCounts {
7877    created: usize,
7878    replaced: usize,
7879    updated: usize,
7880    unchanged: usize,
7881    refused: usize,
7882}
7883
7884impl ScopeOutcomeCounts {
7885    fn of(results: &[ScopeRecordResult]) -> Self {
7886        let mut counts = Self::default();
7887        for result in results {
7888            let slot = match result.outcome {
7889                ScopeRecordOutcome::Created => &mut counts.created,
7890                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7891                ScopeRecordOutcome::Updated => &mut counts.updated,
7892                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7893                ScopeRecordOutcome::Refused => &mut counts.refused,
7894            };
7895            *slot += 1;
7896        }
7897        counts
7898    }
7899}
7900
7901#[cfg(test)]
7902mod scope_outcome_count_tests {
7903    use super::*;
7904
7905    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7906        ScopeRecordResult {
7907            scope_ref: "r".to_string(),
7908            scope_epoch: 1,
7909            outcome,
7910            code: None,
7911            message: None,
7912            version: None,
7913            parent_state: None,
7914        }
7915    }
7916
7917    /// Each outcome lands in its own count, so a refused record can never be
7918    /// hidden inside the total the log already printed.
7919    #[test]
7920    fn every_outcome_is_counted_in_its_own_field() {
7921        let results = [
7922            result(ScopeRecordOutcome::Created),
7923            result(ScopeRecordOutcome::Created),
7924            result(ScopeRecordOutcome::Replaced),
7925            result(ScopeRecordOutcome::Updated),
7926            result(ScopeRecordOutcome::Unchanged),
7927            result(ScopeRecordOutcome::Refused),
7928            result(ScopeRecordOutcome::Refused),
7929            result(ScopeRecordOutcome::Refused),
7930        ];
7931        assert_eq!(
7932            ScopeOutcomeCounts::of(&results),
7933            ScopeOutcomeCounts {
7934                created: 2,
7935                replaced: 1,
7936                updated: 1,
7937                unchanged: 1,
7938                refused: 3,
7939            }
7940        );
7941    }
7942}
7943
7944/// Return whether a catalog change can create a newly violating live route.
7945/// Removing an attested claim is intentionally excluded: it makes fewer routes
7946/// forbidden and therefore must leave the existing route census untouched.
7947fn capability_census_trigger(
7948    old: Option<&CapabilityDeclarations>,
7949    new: Option<&CapabilityDeclarations>,
7950) -> bool {
7951    let old_provides = old
7952        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7953        .unwrap_or_default();
7954    let old_denies = old
7955        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7956        .unwrap_or_default();
7957    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7958        provides: Vec::new(),
7959        requires: Vec::new(),
7960        must_never_reach: Vec::new(),
7961    });
7962
7963    new.provides
7964        .iter()
7965        .any(|capability| !old_provides.contains(capability))
7966        || new
7967            .must_never_reach
7968            .iter()
7969            .any(|capability| !old_denies.contains(capability))
7970}
7971
7972/// Find the first capability an attested opener denies that an attested target
7973/// claims. Both manifests are live registry records, never cached or client data.
7974fn denied_capability<'a>(
7975    opening_manifest: &'a ModuleManifest,
7976    target_manifest: &ModuleManifest,
7977) -> Option<&'a str> {
7978    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7979    let target_capabilities = target_manifest.capabilities.as_ref()?;
7980    opening_capabilities
7981        .must_never_reach
7982        .iter()
7983        .find(|denied| {
7984            target_capabilities
7985                .provides
7986                .iter()
7987                .any(|provided| provided == *denied)
7988        })
7989        .map(String::as_str)
7990}
7991
7992fn catalog_update_frozen_field_message(
7993    registered: &ModuleManifest,
7994    provides: &[ProviderRole],
7995) -> Option<String> {
7996    let old_has_provides = !registered.provides.is_empty();
7997    let new_has_provides = !provides.is_empty();
7998    if old_has_provides != new_has_provides {
7999        return Some(format!(
8000            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8001            registered.module_id
8002        ));
8003    }
8004
8005    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
8006        return Some(format!(
8007            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8008            registered.module_id
8009        ));
8010    }
8011
8012    let registered_concurrency = manifest_concurrency(registered);
8013    let mut candidate = registered.clone();
8014    candidate.provides = provides.to_vec();
8015    let candidate_concurrency = manifest_concurrency(&candidate);
8016    if candidate_concurrency != registered_concurrency {
8017        return Some(format!(
8018            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8019            registered.module_id, registered_concurrency, candidate_concurrency
8020        ));
8021    }
8022
8023    // control_ops live beside the manifest in the HELLO body, not inside
8024    // ModuleManifest, so a provides-only catalog.update cannot change them.
8025    None
8026}
8027
8028fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8029    manifest.provides.iter().any(is_routable_role)
8030}
8031
8032/// Returns the routable-provider concurrency subc should enforce for this manifest.
8033///
8034/// ToolProvider and ManagementSurface store their delivery concurrency directly.
8035/// InternalService has no role-specific concurrency field, so it retains the
8036/// existing ModuleManaged default for backward compatibility.
8037fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8038    manifest
8039        .provides
8040        .iter()
8041        .find_map(|provider| match provider {
8042            ProviderRole::ToolProvider { concurrency, .. }
8043            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8044            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8045        })
8046        .unwrap_or(Concurrency::ModuleManaged)
8047}
8048
8049/// True when the manifest carries a ManagementSurface role whose concurrency
8050/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
8051/// bytes because the typed manifest deliberately erases that distinction: the
8052/// default exists for wire compatibility, and this probe exists so the default
8053/// stays observable. Any parse irregularity returns false -- the caller only
8054/// logs, and a malformed body already failed registration upstream.
8055fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8056    let has_management_surface = manifest
8057        .provides
8058        .iter()
8059        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8060    if !has_management_surface {
8061        return false;
8062    }
8063    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8064        return false;
8065    };
8066    let Some(provides) = raw
8067        .get("manifest")
8068        .and_then(|manifest| manifest.get("provides"))
8069        .and_then(serde_json::Value::as_array)
8070    else {
8071        return false;
8072    };
8073    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
8074    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
8075    // against the management_surface_manifest_without_concurrency golden, not
8076    // recalled (the externally-tagged guess was this function's first bug).
8077    provides.iter().any(|role| {
8078        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8079            && role.get("concurrency").is_none()
8080    })
8081}
8082
8083fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8084    if peer_version != PROTOCOL_VERSION {
8085        return Err(format!(
8086            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8087        ));
8088    }
8089    Ok(PROTOCOL_VERSION)
8090}
8091
8092fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8093    Frame::build_with_version(
8094        response_version(frame),
8095        FrameType::Pong,
8096        frame.header.flags,
8097        0,
8098        0,
8099        frame.header.corr,
8100        Vec::new(),
8101    )
8102    .map_err(RouterError::FrameBuild)
8103}
8104
8105pub(crate) fn control_error_frame(
8106    frame: &Frame,
8107    code: &'static str,
8108    message: impl Into<String>,
8109) -> Result<Frame, RouterError> {
8110    control_error_body_frame(
8111        frame,
8112        ErrorBody {
8113            code: code.to_string(),
8114            message: message.into(),
8115            detail: None,
8116        },
8117    )
8118}
8119
8120/// A channel-0 ERROR for a refusal the daemon decided itself: every
8121/// route.open refusal, the drain `module_reloading` and scope-change/end
8122/// bodies, spawn-subscription and supervisor refusals. Carries the
8123/// daemon-origin bit, which is what lets a client treat `module_reloading`
8124/// as proof that its request never reached a module. Response, push,
8125/// goodbye and relay-request frames keep the plain control flags in their
8126/// own builders.
8127fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8128    control_error_frame_with_flags(frame, error, daemon_error_flags())
8129}
8130
8131/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8132/// refused route.bind answered as a route.open refusal, or a refused
8133/// module-control RPC. Plain control flags — the code and message are
8134/// module-chosen, so the frame must not claim daemon authorship, even when
8135/// the module picks a daemon code like `module_reloading`.
8136fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8137    control_error_frame_with_flags(frame, error, control_flags())
8138}
8139
8140fn control_error_frame_with_flags(
8141    frame: &Frame,
8142    error: ErrorBody,
8143    flags: Flags,
8144) -> Result<Frame, RouterError> {
8145    let body = serde_json::to_vec(&error).map_err(|err| {
8146        RouterError::backend(
8147            0,
8148            frame.header.corr,
8149            format!("failed to encode control ERROR: {err}"),
8150        )
8151    })?;
8152
8153    Frame::build_with_version(
8154        response_version(frame),
8155        FrameType::Error,
8156        flags,
8157        0,
8158        0,
8159        frame.header.corr,
8160        body,
8161    )
8162    .map_err(RouterError::FrameBuild)
8163}
8164
8165pub(crate) fn control_response_body_frame<T: Serialize>(
8166    frame: &Frame,
8167    reply: &T,
8168    label: &'static str,
8169) -> Result<Frame, RouterError> {
8170    let body = serde_json::to_vec(reply).map_err(|err| {
8171        RouterError::backend(
8172            0,
8173            frame.header.corr,
8174            format!("failed to encode {label}: {err}"),
8175        )
8176    })?;
8177
8178    Frame::build_with_version(
8179        response_version(frame),
8180        FrameType::Response,
8181        control_flags(),
8182        0,
8183        0,
8184        frame.header.corr,
8185        body,
8186    )
8187    .map_err(RouterError::FrameBuild)
8188}
8189
8190/// Map a forwarding failure to the wire code a client sees.
8191///
8192/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8193/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8194/// chosen here decides whether a caller retries or gives up.
8195///
8196/// That makes attribution the load-bearing property, not merely having a code. A
8197/// permanent fault published as a retryable one produces a fleet-wide retry storm
8198/// against something that can never recover; a transient fault published as
8199/// permanent gives up on work that would have succeeded. Both look correct in a
8200/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8201/// the mapping per variant rather than merely asserting that some code exists.
8202///
8203/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8204/// two codes on the same side of the boundary passes it. Measured rather than
8205/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8206/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8207/// a test named for something else that happens to assert the string.
8208///
8209/// That accidental coverage is deliberately left alone rather than promoted to a
8210/// named test, because it guards a property this function does not promise.
8211/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8212/// specific code within a class, so identity is free to change and only the
8213/// partition is a contract. Splitting it out would assert a guarantee nothing
8214/// depends on — and a suite that promises more than the code does is the harder
8215/// thing to correct later, because the next reader cannot tell which assertions
8216/// are load-bearing.
8217///
8218/// Pin identity here the moment a consumer branches on a specific code.
8219fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8220    match err {
8221        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8222        ForwardingError::NoModuleConnection => "target_unavailable",
8223        ForwardingError::ModuleReloading { .. } => "module_reloading",
8224        ForwardingError::ClientRouteChannelExhausted { .. }
8225        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8226        ForwardingError::StaleModuleEndpoint
8227        | ForwardingError::UnknownReservation { .. }
8228        | ForwardingError::ConnectionClosing { .. }
8229        | ForwardingError::ClientEgressClosed { .. }
8230        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8231        // Only a swap candidate's registration can produce this, and it means
8232        // exactly what a second active HELLO for a live id means.
8233        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8234        ForwardingError::RelayCorrelationExhausted
8235        | ForwardingError::RouteOpenBuild(_)
8236        | ForwardingError::Poisoned => "forwarding_error",
8237    }
8238}
8239
8240fn response_version(frame: &Frame) -> u8 {
8241    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8242        frame.header.ver
8243    } else {
8244        PROTOCOL_VERSION
8245    }
8246}
8247
8248fn control_flags() -> Flags {
8249    Flags::new(false, Priority::Passive, false)
8250}
8251
8252/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8253/// channel. The target is the module (a client never saw the route), so this
8254/// takes the module path: delivered late rather than dropped when the module's
8255/// queue is momentarily full, and never closing its connection.
8256fn send_goodbye_target_best_effort(
8257    counters: &DaemonCounters,
8258    target: &GoodbyeTarget,
8259    context: &'static str,
8260) {
8261    let Ok(frame) = Frame::build_with_version(
8262        target.negotiated_ver,
8263        FrameType::Goodbye,
8264        control_flags(),
8265        target.channel,
8266        target.epoch,
8267        0,
8268        Vec::new(),
8269    ) else {
8270        return;
8271    };
8272    crate::forwarding::send_module_route_goodbye(
8273        counters,
8274        &target.sink,
8275        frame,
8276        target.module_id.as_deref(),
8277        context,
8278    );
8279}
8280
8281pub(crate) fn send_route_control_pushes(
8282    forwarding: &ForwardingTable,
8283    routes: Vec<EndpointRoute>,
8284    push: ClientControlPush,
8285) {
8286    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8287    for route in routes {
8288        let target = route.goodbye_target;
8289        if let Some((existing, channels)) = targets
8290            .iter_mut()
8291            .find(|(existing, _)| existing.connection_id == target.connection_id)
8292        {
8293            debug_assert_eq!(
8294                existing.negotiated_ver, target.negotiated_ver,
8295                "one connection cannot negotiate multiple frame versions"
8296            );
8297            if !channels.contains(&target.channel) {
8298                channels.push(target.channel);
8299            }
8300            continue;
8301        }
8302        let channel = target.channel;
8303        targets.push((target, vec![channel]));
8304    }
8305    for (target, mut channels) in targets {
8306        channels.sort_unstable();
8307        let mut push = push.clone();
8308        match &mut push {
8309            ClientControlPush::RouteClosing {
8310                channels: covered, ..
8311            }
8312            | ClientControlPush::RouteClosed {
8313                channels: covered, ..
8314            } => *covered = channels,
8315        }
8316        let body = match serde_json::to_vec(&push) {
8317            Ok(body) => body,
8318            Err(err) => {
8319                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8320                continue;
8321            }
8322        };
8323        let frame = match Frame::build_with_version(
8324            target.negotiated_ver,
8325            FrameType::Push,
8326            control_flags(),
8327            0,
8328            0,
8329            0,
8330            body.clone(),
8331        ) {
8332            Ok(frame) => frame,
8333            Err(err) => {
8334                warn!(
8335                    route_channel = target.channel,
8336                    error = %err,
8337                    "failed to build route lifecycle control PUSH frame"
8338                );
8339                continue;
8340            }
8341        };
8342        if let Err(err) = target.sink.try_send(frame) {
8343            if target.close_on_delivery_failure() {
8344                warn!(
8345                    target_connection_id = target.connection_id.get(),
8346                    route_channel = target.channel,
8347                    error = %err,
8348                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8349                );
8350                let _ = forwarding.escalate_client_delivery_failure(
8351                    target.connection_id,
8352                    target.channel,
8353                    target.epoch,
8354                    CloseReason::new(
8355                        "route_lifecycle_push_delivery_failed",
8356                        format!(
8357                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8358                            target.channel
8359                        ),
8360                    ),
8361                    crate::forwarding::UndeliveredFrame {
8362                        module_id: target.module_id.as_deref(),
8363                        sink: &target.sink,
8364                    },
8365                );
8366            }
8367        }
8368    }
8369}
8370
8371#[cfg(test)]
8372mod tests {
8373    mod peer_manifest {
8374        include!("peer_manifest_tests.rs");
8375    }
8376    mod activity_read {
8377        include!("activity_read_tests.rs");
8378    }
8379
8380    mod lifecycle_activity {
8381        include!("control_lifecycle_tests.rs");
8382    }
8383    #[cfg(unix)]
8384    #[tokio::test]
8385    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8386        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8387        let path = dir.join("subc.jsonc");
8388        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8389            "program":"/bin/sleep","args":["60"],"protocol":"none",
8390            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8391        }}}).to_string()).unwrap();
8392        let mut configured = crate::daemon_config::load(&path)
8393            .unwrap()
8394            .unwrap()
8395            .modules
8396            .pop()
8397            .unwrap();
8398        let registry = std::sync::Arc::new(crate::Registry::default());
8399        let handle = crate::SupervisorHandle::new();
8400        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8401            .with_handle(handle.clone());
8402        let module = supervisor
8403            .supervise_configured_with_health(
8404                configured.module_spec(),
8405                true,
8406                configured.health.clone(),
8407                None,
8408                configured.restart,
8409            )
8410            .unwrap();
8411        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8412        let before = module.status().unwrap().pid;
8413        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8414        configured.health.cadence = std::time::Duration::from_secs(3600);
8415        let health_only = handler
8416            .reconcile_supervised_modules(
8417                &supervisor,
8418                vec![configured.clone()],
8419                false,
8420                Principal::Direct,
8421            )
8422            .await
8423            .unwrap();
8424        assert!(
8425            health_only.0.changed_pending_reload.is_empty(),
8426            "health policy is already applied live"
8427        );
8428        assert_eq!(module.status().unwrap().pid, before);
8429        assert_eq!(
8430            module.configuration().unwrap().1.http,
8431            configured.health.http
8432        );
8433        configured.args = vec!["61".into()];
8434        let launch = handler
8435            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8436            .await
8437            .unwrap();
8438        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8439        assert_eq!(
8440            module.status().unwrap().pid,
8441            before,
8442            "a launch edit is stored until reload"
8443        );
8444        module.drain().await.unwrap();
8445    }
8446    use cortexkit_test_support::ScratchDir;
8447    use std::{
8448        collections::BTreeMap,
8449        fmt,
8450        path::PathBuf,
8451        sync::{Arc, Mutex},
8452        time::Duration,
8453    };
8454
8455    use serde_json::{json, Value};
8456    use subc_protocol::{
8457        manifest::{
8458            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8459            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8460        },
8461        session::HealthStatus,
8462        FrameType,
8463    };
8464
8465    use super::*;
8466    use crate::{
8467        forwarding::{DataRoute, DataRouteState},
8468        registry::ChannelState,
8469        router::FrameSink,
8470        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8471        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8472        RouteCtx, Router,
8473    };
8474    use tokio::{
8475        sync::mpsc,
8476        time::{sleep, Instant},
8477    };
8478    use tracing::{
8479        field::{Field, Visit},
8480        Event, Subscriber,
8481    };
8482    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8483
8484    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8485    ///
8486    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8487    /// is only populated for `tests/*.rs` integration test binaries -- this file
8488    /// compiles as part of the library target, which gets neither. This test's
8489    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8490    /// and the sibling binary lives two directories up at
8491    /// `<target-dir>/<profile>/fake-aft-stub`.
8492    ///
8493    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8494    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8495    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8496    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8497    /// `NotFound`, which reads as a broken test rather than an unbuilt
8498    /// dependency -- so state the cause and the remedy instead. Deliberately a
8499    /// panic and not a silent skip: a test that quietly passes when it could not
8500    /// run is worse than one that fails, because it reports health it never
8501    /// verified.
8502    fn fake_aft_stub_path() -> PathBuf {
8503        let mut path = std::env::current_exe().expect("current_exe available in tests");
8504        path.pop(); // .../deps/
8505        path.pop(); // .../<profile>/
8506        path.push(if cfg!(windows) {
8507            "fake-aft-stub.exe"
8508        } else {
8509            "fake-aft-stub"
8510        });
8511        assert!(
8512            path.exists(),
8513            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8514             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8515            path.display()
8516        );
8517        path
8518    }
8519
8520    /// Whether clients retry `code` in place: the predicate itself, never a copy
8521    /// of its set. A copied list breaks silently when a code is added to or
8522    /// removed from the real one, and a stale copy here would let exactly the
8523    /// failure this test exists to catch pass.
8524    fn client_retries(code: &str) -> bool {
8525        subc_protocol::error_codes::is_retryable_route_open(code)
8526    }
8527
8528    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8529    /// of failure is worse than publishing none. A permanent fault dressed as
8530    /// retryable makes every client in the fleet retry forever against something
8531    /// that cannot recover; a transient fault dressed as permanent abandons work
8532    /// that would have succeeded.
8533    ///
8534    /// Asserting "a code exists" cannot catch either, because the string is free
8535    /// to say anything. This enumerates every variant and pins which side of the
8536    /// retry boundary it lands on, so a new variant must be classified here
8537    /// deliberately rather than inheriting whichever arm it was appended to.
8538    #[test]
8539    fn retryability_of_forwarding_codes_matches_the_failure() {
8540        // Transient by nature: the target is booting, reloading, or its endpoint
8541        // was swapped mid-flight. Retrying is how these resolve.
8542        let transient = [
8543            ForwardingError::NoModuleConnection,
8544            ForwardingError::ModuleReloading {
8545                module_id: "m".into(),
8546            },
8547            ForwardingError::StaleModuleEndpoint,
8548            ForwardingError::UnknownReservation {
8549                client_channel: 1,
8550                module_channel: 1,
8551            },
8552            ForwardingError::ConnectionClosing {
8553                connection_id: ConnectionId::new(1),
8554            },
8555            ForwardingError::ClientEgressClosed {
8556                connection_id: ConnectionId::new(1),
8557            },
8558            ForwardingError::ModuleEgressUnavailable {
8559                connection_id: ConnectionId::new(1),
8560            },
8561        ];
8562        for err in transient {
8563            let code = forwarding_error_code(&err);
8564            assert!(
8565                client_retries(code),
8566                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8567            );
8568        }
8569
8570        // Not fixed by retrying. Channel and correlation exhaustion need the
8571        // caller to close routes, and a poisoned lock is a daemon that cannot
8572        // recover at all — the worst thing to advertise as retryable, since every
8573        // client would storm a daemon that will never answer.
8574        let permanent = [
8575            ForwardingError::ConnectionRoleConflict {
8576                connection_id: ConnectionId::new(1),
8577            },
8578            ForwardingError::ClientRouteChannelExhausted {
8579                connection_id: ConnectionId::new(1),
8580            },
8581            ForwardingError::ModuleRouteChannelExhausted {
8582                endpoint: ModuleEndpointId {
8583                    connection_id: ConnectionId::new(1),
8584                    generation: 1,
8585                },
8586            },
8587            ForwardingError::RelayCorrelationExhausted,
8588            ForwardingError::RouteOpenBuild("x".into()),
8589            ForwardingError::Poisoned,
8590        ];
8591        for err in permanent {
8592            let code = forwarding_error_code(&err);
8593            assert!(
8594                !client_retries(code),
8595                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8596            );
8597        }
8598    }
8599
8600    /// The principal is the daemon's answer to "who is calling", and modules
8601    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8602    /// plexus gates connector invocation. So a stamp is an authorization input in
8603    /// another process, not a label — and both possible answers SUCCEED, which is
8604    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8605    /// first-party capability to something that never proved it; a supervised one
8606    /// stamped `Direct` silently strips a module of capability it is entitled to.
8607    ///
8608    /// Neither shows up in a test that only checks the bind succeeded. Before this
8609    /// test the only coverage was accidental —
8610    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8611    /// stamped principal on its way past, so narrowing that wire-shape test to its
8612    /// stated subject would have deleted the last assertion on this value. It
8613    /// still asserts the stamp, which is now redundancy rather than the only
8614    /// guard: both fail under the same mutation, and this one names the reason.
8615    /// SCOPE: this handler's supervisor has spawned nothing, so
8616    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8617    /// is unreachable here. Both assertions below are refusals, and a mutant that
8618    /// refuses everything would satisfy them.
8619    ///
8620    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8621    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8622    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8623    /// at source rather than assumed, since a citation is a claim about another
8624    /// file and ages like one. Recorded because a harness that structurally
8625    /// cannot reach an arm reports "none" for that arm identically to one that
8626    /// covers it and found nothing.
8627    #[tokio::test]
8628    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8629        let handler = ControlHandler::default();
8630        let frame =
8631            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8632
8633        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8634        // any process holding the connection file. Nothing was proved, so nothing
8635        // may be granted beyond the unattested floor.
8636        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8637        assert_eq!(
8638            stamped,
8639            Principal::Direct,
8640            "a caller that proved nothing must not be stamped as a supervised module"
8641        );
8642
8643        // A claimed module_id with a nonce no supervised child was given is a
8644        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8645        // quietly demoted to Direct, or an impersonation attempt looks identical
8646        // to an ordinary unattested connection.
8647        let forged = handler
8648            .route_open_principal(
8649                &frame,
8650                Some(ConsumerIdentity {
8651                    module_id: "aft".to_string(),
8652                    launch_nonce: "not-a-real-nonce".to_string(),
8653                }),
8654            )
8655            .unwrap();
8656        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8657        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8658    }
8659
8660    /// The test above hands `route_open_principal` an identity it built itself,
8661    /// which proves the stamping rule and nothing about where the identity comes
8662    /// from. The real producer is a wire body, and the two are joined by a serde
8663    /// field name that nothing else asserts.
8664    ///
8665    /// That join fails quietly in one specific way: an unrecognised key is simply
8666    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8667    /// yields `None` and every supervised module silently drops to `Direct`.
8668    /// Capability-wise that is the safe direction, but it surfaces far from its
8669    /// cause — as a module mysteriously refused bash — and it would pass every
8670    /// test that builds its own input.
8671    ///
8672    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8673    /// would break every client the moment the daemon gains a field, trading a
8674    /// quiet demotion for a hard refusal on additive change. Asserting the join
8675    /// instead means a rename breaks a test here rather than the fleet.
8676    #[test]
8677    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8678        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8679        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8680        let ClientControlRequest::RouteOpen {
8681            consumer_identity, ..
8682        } = parsed
8683        else {
8684            panic!("route.open body must parse as RouteOpen");
8685        };
8686        assert_eq!(
8687            consumer_identity,
8688            Some(ConsumerIdentity {
8689                module_id: "aft".to_string(),
8690                launch_nonce: "n".to_string(),
8691            }),
8692            "the wire field name must reach the value route_open_principal reads"
8693        );
8694    }
8695
8696    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8697        ModuleManifest::builder(module_id, "0.1.0")
8698            .protocol_ver(protocol_ver)
8699            .provides(vec![ProviderRole::ToolProvider {
8700                tools: vec![Tool {
8701                    name: "read".to_string(),
8702                    description: None,
8703                    execution_mode: ExecutionMode::Pure,
8704                    schema: json!({"type": "object"}),
8705                }],
8706                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8707                concurrency: Concurrency::ModuleManaged,
8708                emits_push: true,
8709                sub_supervises: true,
8710            }])
8711            .build()
8712    }
8713
8714    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8715        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8716    }
8717
8718    fn hello_frame_with_control_ops(
8719        module_id: &str,
8720        protocol_ver: u8,
8721        corr: u64,
8722        control_ops: Option<Vec<String>>,
8723    ) -> Frame {
8724        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8725    }
8726
8727    fn hello_frame_with_nonce(
8728        module_id: &str,
8729        protocol_ver: u8,
8730        corr: u64,
8731        launch_nonce: Option<&str>,
8732    ) -> Frame {
8733        hello_frame_full(
8734            module_id,
8735            protocol_ver,
8736            corr,
8737            None,
8738            launch_nonce.map(ToOwned::to_owned),
8739        )
8740    }
8741
8742    fn hello_frame_full(
8743        module_id: &str,
8744        protocol_ver: u8,
8745        corr: u64,
8746        control_ops: Option<Vec<String>>,
8747        launch_nonce: Option<String>,
8748    ) -> Frame {
8749        let body = serde_json::to_vec(&ModuleHelloBody {
8750            manifest: manifest(module_id, protocol_ver),
8751            protocol_ver,
8752            control_ops,
8753            launch_nonce,
8754        })
8755        .unwrap();
8756        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8757    }
8758
8759    fn hello_frame_with_rich_ui(
8760        module_id: &str,
8761        protocol_ver: u8,
8762        corr: u64,
8763        launch_nonce: Option<&str>,
8764        rich_ui: Value,
8765    ) -> Frame {
8766        let mut body = serde_json::to_value(ModuleHelloBody {
8767            manifest: manifest(module_id, protocol_ver),
8768            protocol_ver,
8769            control_ops: None,
8770            launch_nonce: launch_nonce.map(str::to_string),
8771        })
8772        .unwrap();
8773        body["manifest"]["rich_ui"] = rich_ui;
8774        Frame::build(
8775            FrameType::Hello,
8776            control_flags(),
8777            0,
8778            0,
8779            corr,
8780            serde_json::to_vec(&body).unwrap(),
8781        )
8782        .unwrap()
8783    }
8784
8785    fn non_routable_hello_frame_with_control_ops(
8786        module_id: &str,
8787        corr: u64,
8788        control_ops: Option<Vec<String>>,
8789    ) -> Frame {
8790        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8791        manifest.provides.clear();
8792        let body = serde_json::to_vec(&ModuleHelloBody {
8793            manifest,
8794            protocol_ver: PROTOCOL_VERSION,
8795            control_ops,
8796            launch_nonce: None,
8797        })
8798        .unwrap();
8799        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8800    }
8801
8802    fn capability_grammar_hello_frame(
8803        capabilities: Value,
8804        runtime_computed: Option<Value>,
8805        corr: u64,
8806    ) -> Frame {
8807        let mut body = serde_json::to_value(ModuleHelloBody {
8808            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8809            protocol_ver: PROTOCOL_VERSION,
8810            control_ops: None,
8811            launch_nonce: None,
8812        })
8813        .expect("HELLO body serializes");
8814        body["manifest"]["capabilities"] = capabilities;
8815        if let Some(runtime_computed) = runtime_computed {
8816            body["runtime_computed"] = runtime_computed;
8817        }
8818        Frame::build(
8819            FrameType::Hello,
8820            control_flags(),
8821            0,
8822            0,
8823            corr,
8824            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8825        )
8826        .expect("HELLO frame builds")
8827    }
8828
8829    fn channel_request(channel: u16, corr: u64) -> Frame {
8830        Frame::build(
8831            FrameType::Request,
8832            Flags::new(true, Priority::Interactive, false),
8833            channel,
8834            0,
8835            corr,
8836            b"opaque".to_vec(),
8837        )
8838        .unwrap()
8839    }
8840
8841    fn route_ctx(
8842        connection_id: ConnectionId,
8843    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8844        let (tx, rx) = mpsc::channel(8);
8845        (
8846            RouteCtx {
8847                connection_id,
8848                egress: FrameSink::new(tx),
8849            },
8850            rx,
8851        )
8852    }
8853
8854    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8855        serde_json::from_slice(&frame.body).unwrap()
8856    }
8857
8858    /// Register a module over a connection that has a sink and return the
8859    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8860    /// module's own sink rather than returning it as a reply, so the ack is
8861    /// taken off `rx` here and whatever the test reads next is what followed it.
8862    async fn hello_via_sink(
8863        handler: &ControlHandler,
8864        ctx: &RouteCtx,
8865        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8866        hello: Frame,
8867    ) -> Frame {
8868        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8869        assert!(
8870            replies.is_empty(),
8871            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8872        );
8873        let ack = rx
8874            .try_recv()
8875            .expect("HELLO_ACK is queued on the module sink")
8876            .frame;
8877        assert_eq!(ack.header.ty, FrameType::HelloAck);
8878        ack
8879    }
8880
8881    fn parse_error(frame: &Frame) -> Value {
8882        serde_json::from_slice(&frame.body).unwrap()
8883    }
8884
8885    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8886        serde_json::from_slice(&frame.body).unwrap()
8887    }
8888
8889    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8890        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8891            route_channel,
8892            route_epoch: 0,
8893            kind,
8894        })
8895        .unwrap();
8896        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8897    }
8898
8899    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8900        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8901            module_id: module_id.to_string(),
8902        })
8903        .unwrap();
8904        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8905    }
8906
8907    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8908        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8909    }
8910
8911    fn route_open_frame_with_consumer_capabilities(
8912        corr: u64,
8913        module_id: &str,
8914        project_root: ScratchDir,
8915        consumer_capabilities: Option<Vec<String>>,
8916    ) -> Frame {
8917        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8918            target: RouteTarget::ToolProvider {
8919                module_id: module_id.to_string(),
8920            },
8921            identity: BindIdentity::new(
8922                project_root.path().to_path_buf(),
8923                "unit".to_string(),
8924                "session".to_string(),
8925            ),
8926            consumer_identity: None,
8927            consumer_capabilities,
8928            role_versions: None,
8929            admission_facts: None,
8930            scope: None,
8931        })
8932        .unwrap();
8933        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8934    }
8935
8936    fn route_open_frame_with_role_versions(
8937        corr: u64,
8938        module_id: &str,
8939        project_root: ScratchDir,
8940        role_versions: Option<BTreeMap<String, String>>,
8941    ) -> Frame {
8942        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8943            target: RouteTarget::ToolProvider {
8944                module_id: module_id.to_string(),
8945            },
8946            identity: BindIdentity::new(
8947                project_root.path().to_path_buf(),
8948                "unit".to_string(),
8949                format!("session-{corr}"),
8950            ),
8951            consumer_identity: None,
8952            consumer_capabilities: None,
8953            role_versions,
8954            admission_facts: None,
8955            scope: None,
8956        })
8957        .unwrap();
8958        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8959    }
8960
8961    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8962        entries
8963            .iter()
8964            .map(|(role, version)| (role.to_string(), version.to_string()))
8965            .collect()
8966    }
8967
8968    fn route_open_frame_with_admission_facts(
8969        corr: u64,
8970        module_id: &str,
8971        project_root: ScratchDir,
8972        consumer_identity: Option<subc_control::ConsumerIdentity>,
8973        facts: Option<Value>,
8974    ) -> Frame {
8975        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8976            target: RouteTarget::ToolProvider {
8977                module_id: module_id.to_string(),
8978            },
8979            identity: BindIdentity::new(
8980                project_root.path().to_path_buf(),
8981                "unit".to_string(),
8982                format!("session-{corr}"),
8983            ),
8984            consumer_identity,
8985            consumer_capabilities: None,
8986            role_versions: None,
8987            admission_facts: facts,
8988            scope: None,
8989        })
8990        .unwrap();
8991        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8992    }
8993
8994    #[derive(Clone, Default)]
8995    struct EventCapture {
8996        events: Arc<Mutex<Vec<CapturedEvent>>>,
8997    }
8998
8999    #[derive(Clone, Debug)]
9000    struct CapturedEvent {
9001        target: String,
9002        level: tracing::Level,
9003        fields: BTreeMap<String, String>,
9004    }
9005
9006    impl EventCapture {
9007        fn events(&self) -> Vec<CapturedEvent> {
9008            self.events.lock().unwrap().clone()
9009        }
9010    }
9011
9012    impl<S> Layer<S> for EventCapture
9013    where
9014        S: Subscriber,
9015    {
9016        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9017            let mut visitor = EventFieldVisitor::default();
9018            event.record(&mut visitor);
9019            self.events.lock().unwrap().push(CapturedEvent {
9020                target: event.metadata().target().to_string(),
9021                level: *event.metadata().level(),
9022                fields: visitor.fields,
9023            });
9024        }
9025    }
9026
9027    #[derive(Default)]
9028    struct EventFieldVisitor {
9029        fields: BTreeMap<String, String>,
9030    }
9031
9032    impl Visit for EventFieldVisitor {
9033        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9034            self.fields
9035                .insert(field.name().to_string(), format!("{value:?}"));
9036        }
9037    }
9038
9039    fn health_response(corr: u64, status: HealthStatus) -> Frame {
9040        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9041            status,
9042            detail: Some("warming".to_string()),
9043            metrics: Some(json!({"queue_depth": 3})),
9044        })
9045        .unwrap();
9046        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9047    }
9048
9049    fn route_bind_ack(corr: u64) -> Frame {
9050        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9051        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9052    }
9053
9054    fn unique_project_root(label: &str) -> ScratchDir {
9055        ScratchDir::new(label)
9056    }
9057
9058    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9059        match parse_route_poll(frame) {
9060            ClientControlResponse::RoutePoll {
9061                status: None,
9062                live: Some(live),
9063                ..
9064            } => assert_eq!(live, expected_live),
9065            other => panic!("unexpected route.poll response: {other:?}"),
9066        }
9067    }
9068
9069    fn bind_liveness_route(
9070        registry: &Registry,
9071        forwarding: &ForwardingTable,
9072        module_id: &str,
9073    ) -> (RouteCtx, u16, u32) {
9074        let module_connection = ConnectionId::new(101);
9075        let client_connection = ConnectionId::new(202);
9076        let registration = registry
9077            .register_with_control_ops(
9078                manifest(module_id, PROTOCOL_VERSION),
9079                PROTOCOL_VERSION,
9080                module_connection,
9081                module_baseline_control_ops(),
9082            )
9083            .unwrap();
9084        let (module_tx, _module_rx) = mpsc::channel(8);
9085        let endpoint = forwarding
9086            .register_module_connection(
9087                module_connection,
9088                module_id.to_string(),
9089                PROTOCOL_VERSION,
9090                manifest_concurrency(&registration.manifest),
9091                FrameSink::new(module_tx),
9092            )
9093            .unwrap();
9094        let (client_ctx, _client_rx) = route_ctx(client_connection);
9095        let pending = forwarding
9096            .begin_route_bind_relay_for_test(
9097                client_connection,
9098                client_ctx.egress.clone(),
9099                1,
9100                module_id,
9101            )
9102            .unwrap();
9103        assert_eq!(pending.endpoint, endpoint);
9104        let route_channel = pending.client_channel;
9105        let route_epoch = pending.client_epoch;
9106        forwarding
9107            .complete_pending_relay(
9108                module_connection,
9109                pending.corr,
9110                RouteBindRelayOutcome::Accepted,
9111            )
9112            .unwrap();
9113        (client_ctx, route_channel, route_epoch)
9114    }
9115
9116    struct FakeProcessLiveness {
9117        live: Option<bool>,
9118    }
9119
9120    impl ModuleProcessLiveness for FakeProcessLiveness {
9121        fn process_live(&self, _module_id: &str) -> Option<bool> {
9122            self.live
9123        }
9124    }
9125
9126    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9127    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9128    {
9129        let registry = Arc::new(Registry::default());
9130        let supervisor_handle = SupervisorHandle::new();
9131        let supervisor = Supervisor::new_for_test(
9132            Arc::clone(&registry),
9133            RestartPolicy::new(1, Duration::from_millis(10)),
9134        )
9135        .with_handle(supervisor_handle.clone());
9136        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9137        let module = supervisor
9138            .spawn(ModuleSpec {
9139                module_id: "stderr-tail-wire".to_string(),
9140                program: fake_aft_stub_path(),
9141                args: Vec::new(),
9142                env: vec![
9143                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9144                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9145                ],
9146                reserved: false,
9147                reserved_prefixes: Vec::new(),
9148                protocol: ModuleProtocol::Subc,
9149                overlap: Default::default(),
9150            })
9151            .unwrap();
9152
9153        let deadline = Instant::now() + Duration::from_secs(5);
9154        loop {
9155            let tail = module.stderr_tail(None, None);
9156            if tail
9157                .entries
9158                .iter()
9159                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9160                && tail.entries.iter().any(|entry| {
9161                    matches!(
9162                        entry,
9163                        TailEntry::Line {
9164                            truncated: true,
9165                            ..
9166                        }
9167                    )
9168                })
9169            {
9170                break;
9171            }
9172            assert!(
9173                Instant::now() < deadline,
9174                "module did not produce a truncated line and restart boundary: {tail:?}"
9175            );
9176            sleep(Duration::from_millis(10)).await;
9177        }
9178
9179        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9180        let request = ClientControlRequest::SupervisorStderrTail {
9181            module_id: "stderr-tail-wire".to_string(),
9182            max_lines: None,
9183            max_bytes: None,
9184        };
9185        let frame = Frame::build(
9186            FrameType::Request,
9187            control_flags(),
9188            0,
9189            0,
9190            1,
9191            serde_json::to_vec(&request).unwrap(),
9192        )
9193        .unwrap();
9194        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9195        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9196        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9197            serde_json::from_slice(&responses[0].body).unwrap()
9198        else {
9199            panic!("expected supervisor.stderr_tail response");
9200        };
9201
9202        assert!(
9203            tail.entries
9204                .iter()
9205                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9206            "the control response lost the restart boundary"
9207        );
9208        let Some(StderrTailEntry::Line {
9209            text,
9210            truncated,
9211            at_ms,
9212        }) = tail.entries.iter().find(|entry| {
9213            matches!(
9214                entry,
9215                StderrTailEntry::Line {
9216                    truncated: true,
9217                    ..
9218                }
9219            )
9220        })
9221        else {
9222            panic!("the control response lost the truncated line");
9223        };
9224        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9225        assert!(*truncated);
9226        assert!(
9227            at_ms.is_some(),
9228            "the control response lost the line's capture time"
9229        );
9230    }
9231
9232    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9233    /// read done on the worker thread would stall every other task until it
9234    /// finished; the read must run off the worker so this test's own task keeps
9235    /// running while the read is paused.
9236    #[tokio::test(flavor = "current_thread")]
9237    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9238        let dir = ScratchDir::new("terminals-off-worker");
9239        let journal_path = dir.join("terminals.jsonl");
9240        let registry = Arc::new(Registry::default());
9241        let supervisor_handle = SupervisorHandle::new();
9242        let supervisor =
9243            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9244                .with_handle(supervisor_handle.clone())
9245                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9246        let module = supervisor
9247            .spawn(ModuleSpec {
9248                module_id: "terminal-off-worker".to_string(),
9249                program: fake_aft_stub_path(),
9250                args: Vec::new(),
9251                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9252                reserved: false,
9253                reserved_prefixes: Vec::new(),
9254                protocol: ModuleProtocol::Subc,
9255                overlap: Default::default(),
9256            })
9257            .unwrap();
9258        let deadline = Instant::now() + Duration::from_secs(5);
9259        while module.terminal_history().entries.len() != 2 {
9260            assert!(Instant::now() < deadline, "module did not record two exits");
9261            sleep(Duration::from_millis(10)).await;
9262        }
9263
9264        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9265        let handler =
9266            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9267        let frame = Frame::build(
9268            FrameType::Request,
9269            control_flags(),
9270            0,
9271            0,
9272            1,
9273            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9274                module_id: "terminal-off-worker".to_string(),
9275            })
9276            .unwrap(),
9277        )
9278        .unwrap();
9279        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9280        let spawned_at = std::time::Instant::now();
9281        let read = tokio::spawn({
9282            let handler = Arc::clone(&handler);
9283            async move { handler.handle_control_frame(&ctx, frame).await }
9284        });
9285        // Waiting for the pause from a blocking thread keeps this task pending,
9286        // so the runtime's single worker is free to run the read task.
9287        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9288            .await
9289            .unwrap()
9290            .expect("the history read reached its pause");
9291        let elapsed = spawned_at.elapsed();
9292        assert!(
9293            elapsed < Duration::from_secs(2) && !read.is_finished(),
9294            "this task could not run while the history read was paused \
9295             (resumed after {elapsed:?}, read finished: {})",
9296            read.is_finished()
9297        );
9298
9299        drop(release);
9300        let responses = read.await.unwrap().unwrap();
9301        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9302        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9303            panic!("expected supervisor.terminals response");
9304        };
9305        assert_eq!(terminals.entries.len(), 2);
9306        assert_eq!(terminals.journal_skipped_lines, 0);
9307        assert_eq!(terminals.journal_read_errors, 0);
9308    }
9309
9310    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9311    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9312        let registry = Arc::new(Registry::default());
9313        let supervisor_handle = SupervisorHandle::new();
9314        let supervisor =
9315            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9316                .with_handle(supervisor_handle.clone());
9317        let module = supervisor
9318            .spawn(ModuleSpec {
9319                module_id: "terminal-golden".to_string(),
9320                program: fake_aft_stub_path(),
9321                args: Vec::new(),
9322                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9323                reserved: false,
9324                reserved_prefixes: Vec::new(),
9325                protocol: ModuleProtocol::Subc,
9326                overlap: Default::default(),
9327            })
9328            .unwrap();
9329
9330        let deadline = Instant::now() + Duration::from_secs(5);
9331        while module.terminal_history().entries.len() != 2 {
9332            assert!(
9333                Instant::now() < deadline,
9334                "module did not retain two terminal exits: {:?}",
9335                module.terminal_history()
9336            );
9337            sleep(Duration::from_millis(10)).await;
9338        }
9339
9340        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9341        let request = ClientControlRequest::SupervisorTerminals {
9342            module_id: "terminal-golden".to_string(),
9343        };
9344        let frame = Frame::build(
9345            FrameType::Request,
9346            control_flags(),
9347            0,
9348            0,
9349            1,
9350            serde_json::to_vec(&request).unwrap(),
9351        )
9352        .unwrap();
9353        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9354        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9355        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9356        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9357            panic!("expected supervisor.terminals response");
9358        };
9359        assert_eq!(terminals.entries.len(), 2);
9360        assert_eq!(terminals.dropped, 0);
9361
9362        let mut rendered = serde_json::to_value(response).unwrap();
9363        // Wall-clock fields are the observation contract, but not stable fixture
9364        // bytes; normalize only them after the real handler has shaped the response.
9365        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9366        for (index, entry) in rendered["entries"]
9367            .as_array_mut()
9368            .expect("terminal response entries array")
9369            .iter_mut()
9370            .enumerate()
9371        {
9372            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9373        }
9374
9375        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9376            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9377        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9378        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9379            std::fs::write(&golden_path, &serialized).unwrap();
9380        }
9381        let expected: Value =
9382            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9383        assert_eq!(rendered, expected);
9384    }
9385
9386    #[test]
9387    fn hello_registers_manifest_and_returns_ack() {
9388        let registry = Arc::new(Registry::default());
9389        let handler = ControlHandler::new(Arc::clone(&registry));
9390        let conn = ConnectionId::new(1);
9391
9392        let responses = handler
9393            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9394            .unwrap();
9395
9396        assert_eq!(responses.len(), 1);
9397        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9398        assert_eq!(responses[0].header.channel, 0);
9399        assert_eq!(responses[0].header.corr, 7);
9400        let ack = parse_ack(&responses[0]);
9401        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9402        assert!(ack
9403            .subc_capabilities
9404            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9405        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9406        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9407        assert!(ack
9408            .subc_ops
9409            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9410        assert!(ack
9411            .subc_ops
9412            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9413
9414        let registration = registry.get_module("aft").unwrap().unwrap();
9415        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9416        assert_eq!(registration.state, ChannelState::Active);
9417        assert_eq!(registration.connection_id, conn);
9418        assert_eq!(registration.control_ops, module_baseline_control_ops());
9419    }
9420
9421    #[test]
9422    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9423        let invalid_identifiers = [
9424            ("case_change", "credentials-Provider/v1"),
9425            ("leading_zero", "credentials-provider/v01"),
9426            ("trailing_hyphen", "credentials-provider-/v1"),
9427            ("consecutive_hyphens", "credentials--provider/v1"),
9428            ("uppercase", "Credentials-provider/v1"),
9429            ("missing_v", "credentials-provider/1"),
9430            ("whitespace", "credentials provider/v1"),
9431            ("zero_version", "credentials-provider/v0"),
9432            ("out_of_range_version", "credentials-provider/v4294967296"),
9433            (
9434                "overlength_name",
9435                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9436            ),
9437        ];
9438        let mut cases = invalid_identifiers
9439            .into_iter()
9440            .map(|(name, identifier)| {
9441                (
9442                    format!("identifier_{name}"),
9443                    "capabilities.provides[0]".to_string(),
9444                    identifier.to_string(),
9445                    json!({ "provides": [identifier] }),
9446                    None,
9447                )
9448            })
9449            .collect::<Vec<_>>();
9450        cases.extend([
9451            (
9452                "unknown_need".to_string(),
9453                "capabilities.requires[0].need".to_string(),
9454                "deferred".to_string(),
9455                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9456                None,
9457            ),
9458            (
9459                "duplicate_provides".to_string(),
9460                "capabilities.provides[1]".to_string(),
9461                "credentials-provider/v1".to_string(),
9462                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9463                None,
9464            ),
9465            (
9466                "duplicate_must_never_reach".to_string(),
9467                "capabilities.must_never_reach[1]".to_string(),
9468                "credentials-provider/v1".to_string(),
9469                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9470                None,
9471            ),
9472            (
9473                "duplicate_requires_same_need".to_string(),
9474                "capabilities.requires[1]".to_string(),
9475                "credentials-provider/v1".to_string(),
9476                json!({ "requires": [
9477                    { "capability": "credentials-provider/v1", "need": "required" },
9478                    { "capability": "credentials-provider/v1", "need": "required" }
9479                ] }),
9480                None,
9481            ),
9482            (
9483                "duplicate_requires_conflicting_need".to_string(),
9484                "capabilities.requires[1]".to_string(),
9485                "credentials-provider/v1".to_string(),
9486                json!({ "requires": [
9487                    { "capability": "credentials-provider/v1", "need": "required" },
9488                    { "capability": "credentials-provider/v1", "need": "optional" }
9489                ] }),
9490                None,
9491            ),
9492            (
9493                "capabilities_root_pointer".to_string(),
9494                "runtime_computed[0]".to_string(),
9495                "/capabilities".to_string(),
9496                json!({}),
9497                Some(json!(["/capabilities"])),
9498            ),
9499            (
9500                "capabilities_descendant_pointer".to_string(),
9501                "runtime_computed[0]".to_string(),
9502                "/capabilities/provides".to_string(),
9503                json!({}),
9504                Some(json!(["/capabilities/provides"])),
9505            ),
9506            (
9507                "malformed_pointer_without_leading_slash".to_string(),
9508                "runtime_computed[0]".to_string(),
9509                "capabilities".to_string(),
9510                json!({}),
9511                Some(json!(["capabilities"])),
9512            ),
9513            (
9514                "malformed_pointer_escape".to_string(),
9515                "runtime_computed[0]".to_string(),
9516                "/roles/~2/tools".to_string(),
9517                json!({}),
9518                Some(json!(["/roles/~2/tools"])),
9519            ),
9520            (
9521                "unknown_capabilities_field".to_string(),
9522                "capabilities.future".to_string(),
9523                "<array>".to_string(),
9524                json!({ "future": [] }),
9525                None,
9526            ),
9527        ]);
9528
9529        for (index, (name, field, value, capabilities, runtime_computed)) in
9530            cases.into_iter().enumerate()
9531        {
9532            let registry = Arc::new(Registry::default());
9533            let handler = ControlHandler::new(Arc::clone(&registry));
9534            let response = handler
9535                .handle_control(
9536                    ConnectionId::new((index + 1) as u64),
9537                    capability_grammar_hello_frame(
9538                        capabilities,
9539                        runtime_computed,
9540                        index as u64 + 1,
9541                    ),
9542                )
9543                .expect("invalid HELLO returns a refusal");
9544
9545            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9546            let error = parse_error(&response[0]);
9547            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9548            let message = error["message"]
9549                .as_str()
9550                .expect("error message is a string");
9551            assert!(
9552                message.contains(&field),
9553                "{name}: field missing from {message}"
9554            );
9555            assert!(
9556                message.contains(&value),
9557                "{name}: value missing from {message}"
9558            );
9559            assert_eq!(
9560                registry
9561                    .active_registration_count()
9562                    .expect("registry reads"),
9563                0,
9564                "{name}: refused HELLO must not create a catalog entry"
9565            );
9566        }
9567    }
9568
9569    #[test]
9570    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9571        let registry = Arc::new(Registry::default());
9572        let handler = ControlHandler::new(Arc::clone(&registry));
9573        let capabilities = json!({
9574            "provides": ["credentials-provider/v1"],
9575            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9576            "must_never_reach": ["federation-transport/v1"]
9577        });
9578        let response = handler
9579            .handle_control(
9580                ConnectionId::new(99),
9581                capability_grammar_hello_frame(
9582                    capabilities.clone(),
9583                    Some(json!(["/roles/0/tools"])),
9584                    99,
9585                ),
9586            )
9587            .expect("valid HELLO registers");
9588        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9589
9590        let request = Frame::build(
9591            FrameType::Request,
9592            control_flags(),
9593            0,
9594            0,
9595            100,
9596            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9597                .expect("catalog request serializes"),
9598        )
9599        .expect("catalog request frame builds");
9600        let response = handler
9601            .handle_catalog_list(request, None)
9602            .expect("catalog list succeeds");
9603        let ClientControlResponse::CatalogList { modules, .. } =
9604            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9605        else {
9606            panic!("catalog request must return catalog.list");
9607        };
9608        assert_eq!(modules.len(), 1);
9609        assert_eq!(
9610            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9611            capabilities
9612        );
9613    }
9614
9615    #[test]
9616    fn catalog_list_mirrors_management_operation_description() {
9617        let registry = Arc::new(Registry::default());
9618        let handler = ControlHandler::new(Arc::clone(&registry));
9619        let description = "List managed records and return their identifiers and metadata.";
9620        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9621        manifest.provides = vec![ProviderRole::ManagementSurface {
9622            operations: vec![ManagementOperation {
9623                name: "records.list".to_string(),
9624                kind: ManagementOperationKind::Query,
9625                description: Some(description.to_string()),
9626            }],
9627            config_schema: json!({"type": "object"}),
9628            observability: vec![ObservabilitySurface {
9629                name: "records.stats".to_string(),
9630                kind: ObservabilityKind::Snapshot,
9631            }],
9632            identity_scope: vec![IdentityScope::Project],
9633            concurrency: Concurrency::ModuleManaged,
9634        }];
9635        registry
9636            .register_with_control_ops(
9637                manifest,
9638                PROTOCOL_VERSION,
9639                ConnectionId::new(99),
9640                Vec::new(),
9641            )
9642            .expect("described management manifest registers");
9643
9644        let request = Frame::build(
9645            FrameType::Request,
9646            control_flags(),
9647            0,
9648            0,
9649            100,
9650            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9651                .expect("catalog request serializes"),
9652        )
9653        .expect("catalog request frame builds");
9654        let response = handler
9655            .handle_catalog_list(request, None)
9656            .expect("catalog list succeeds");
9657        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9658        assert_eq!(
9659            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9660            "catalog.list must preserve the declared operation description verbatim"
9661        );
9662    }
9663
9664    #[test]
9665    fn catalog_list_mirrors_registered_operation_delivery() {
9666        let registry = Arc::new(Registry::default());
9667        let handler = ControlHandler::new(Arc::clone(&registry));
9668        let delivery =
9669            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9670        let mut body = serde_json::to_value(ModuleHelloBody {
9671            manifest: manifest("watch-management", PROTOCOL_VERSION),
9672            protocol_ver: PROTOCOL_VERSION,
9673            control_ops: None,
9674            launch_nonce: None,
9675        })
9676        .unwrap();
9677        body["manifest"]["provides"] = json!([{
9678            "role": "management_surface",
9679            "operations": [
9680                {"name": "status", "kind": "query"},
9681                {"name": "future", "kind": "query"},
9682                {"name": "reset", "kind": "mutate"}
9683            ],
9684            "config_schema": {}, "observability": [], "identity_scope": []
9685        }]);
9686        body["manifest"]["operation_delivery"] = delivery.clone();
9687        let hello = Frame::build(
9688            FrameType::Hello,
9689            control_flags(),
9690            0,
9691            0,
9692            99,
9693            serde_json::to_vec(&body).unwrap(),
9694        )
9695        .unwrap();
9696        let response = handler
9697            .handle_control(ConnectionId::new(99), hello)
9698            .unwrap();
9699        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9700
9701        let (_, registrations) = registry.list_modules().unwrap();
9702        let registered = &registrations[0].manifest;
9703        assert!(registered.is_watch_operation("status"));
9704        assert!(!registered.is_watch_operation("future"));
9705        assert!(!registered.is_watch_operation("reset"));
9706        assert!(!registered.is_watch_operation("missing"));
9707
9708        let request = Frame::build(
9709            FrameType::Request,
9710            control_flags(),
9711            0,
9712            0,
9713            100,
9714            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9715        )
9716        .unwrap();
9717        let response = handler.handle_catalog_list(request, None).unwrap();
9718        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9719        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9720        let ClientControlResponse::CatalogList { modules, .. } =
9721            serde_json::from_slice(&response[0].body).unwrap()
9722        else {
9723            panic!("expected catalog.list");
9724        };
9725        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9726    }
9727
9728    #[test]
9729    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9730        let registry = Arc::new(Registry::default());
9731        let supervisor = SupervisorHandle::new();
9732        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9733        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9734        let descriptor = json!({
9735            "schema": "ckios.richui.manifest/1",
9736            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9737            "bytes": 4096,
9738            "chunks": 4
9739        });
9740        let connection = ConnectionId::new(201);
9741        let hello = handler
9742            .handle_control(
9743                connection,
9744                hello_frame_with_rich_ui(
9745                    "vault",
9746                    PROTOCOL_VERSION,
9747                    201,
9748                    Some("the-real-nonce"),
9749                    descriptor.clone(),
9750                ),
9751            )
9752            .unwrap();
9753        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9754
9755        let request =
9756            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9757        let provides = registry
9758            .get_module_by_connection(connection)
9759            .unwrap()
9760            .unwrap()
9761            .manifest
9762            .provides;
9763        handler
9764            .handle_catalog_update(connection, request, provides, None, Some(false))
9765            .unwrap();
9766
9767        let request = Frame::build(
9768            FrameType::Request,
9769            control_flags(),
9770            0,
9771            0,
9772            203,
9773            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9774        )
9775        .unwrap();
9776        let response = handler.handle_catalog_list(request, None).unwrap();
9777        let ClientControlResponse::CatalogList { modules, .. } =
9778            serde_json::from_slice(&response[0].body).unwrap()
9779        else {
9780            panic!("expected catalog.list");
9781        };
9782        assert_eq!(
9783            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9784            descriptor
9785        );
9786        assert!(modules[0].nonce_verified);
9787    }
9788
9789    fn hello_with_role_declarations(declarations: Value) -> Frame {
9790        let mut body = serde_json::to_value(ModuleHelloBody {
9791            manifest: manifest("role-test", PROTOCOL_VERSION),
9792            protocol_ver: PROTOCOL_VERSION,
9793            control_ops: None,
9794            launch_nonce: None,
9795        })
9796        .unwrap();
9797        body["manifest"]["role_declarations"] = declarations;
9798        Frame::build(
9799            FrameType::Hello,
9800            control_flags(),
9801            0,
9802            0,
9803            301,
9804            serde_json::to_vec(&body).unwrap(),
9805        )
9806        .unwrap()
9807    }
9808
9809    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9810        let request = Frame::build(
9811            FrameType::Request,
9812            control_flags(),
9813            0,
9814            0,
9815            302,
9816            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9817        )
9818        .unwrap();
9819        let response = handler.handle_catalog_list(request, None).unwrap();
9820        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9821        let ClientControlResponse::CatalogList { mut modules, .. } =
9822            serde_json::from_slice(&response[0].body).unwrap()
9823        else {
9824            panic!("expected catalog.list");
9825        };
9826        (raw["modules"][0].clone(), modules.remove(0))
9827    }
9828
9829    #[test]
9830    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9831        let registry = Arc::new(Registry::default());
9832        let handler = ControlHandler::new(Arc::clone(&registry));
9833        let declarations = json!({
9834            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9835            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9836        });
9837        let connection = ConnectionId::new(301);
9838        let response = handler
9839            .handle_control(
9840                connection,
9841                hello_with_role_declarations(declarations.clone()),
9842            )
9843            .unwrap();
9844        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9845        let registered = registry
9846            .get_module_by_connection(connection)
9847            .unwrap()
9848            .unwrap()
9849            .manifest;
9850        // Compared as JSON values, not bytes: object key order is not part of
9851        // the relay's promise, and workspace builds unify serde_json's
9852        // preserve_order feature, which changes the order a literal keeps.
9853        assert_eq!(
9854            serde_json::to_value(&registered.role_declarations).unwrap(),
9855            declarations
9856        );
9857        let (raw, entry) = role_declarations_catalog(&handler);
9858        assert_eq!(raw["role_declarations"], declarations);
9859        assert_eq!(entry.role_declarations, registered.role_declarations);
9860
9861        let request =
9862            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9863        handler
9864            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9865            .unwrap();
9866        let (raw, entry) = role_declarations_catalog(&handler);
9867        assert_eq!(raw["role_declarations"], declarations);
9868        assert_eq!(entry.role_declarations, registered.role_declarations);
9869    }
9870
9871    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9872        let registry = Arc::new(Registry::default());
9873        let handler = ControlHandler::new(Arc::clone(&registry));
9874        let capture = EventCapture::default();
9875        let _subscriber =
9876            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9877        let response = handler
9878            .handle_control(
9879                ConnectionId::new(301),
9880                hello_with_role_declarations(declarations),
9881            )
9882            .unwrap();
9883        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9884        assert!(registry
9885            .get_module_by_connection(ConnectionId::new(301))
9886            .unwrap()
9887            .unwrap()
9888            .manifest
9889            .role_declarations
9890            .is_empty());
9891        let (raw, entry) = role_declarations_catalog(&handler);
9892        assert!(raw.get("role_declarations").is_none());
9893        assert!(entry.role_declarations.is_empty());
9894        let warnings: Vec<_> = capture
9895            .events()
9896            .into_iter()
9897            .filter(|event| {
9898                event.fields.get("message").map(String::as_str)
9899                    == Some("dropping whole role_declarations map from HELLO")
9900            })
9901            .collect();
9902        assert_eq!(warnings.len(), 1);
9903        assert_eq!(warnings[0].level, tracing::Level::WARN);
9904        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9905        assert_eq!(warnings[0].fields["reason"], reason);
9906    }
9907
9908    #[test]
9909    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9910        for value in [
9911            json!(null),
9912            json!(false),
9913            json!(0),
9914            json!("opaque"),
9915            json!([]),
9916        ] {
9917            assert_hello_drops_role_declarations_with_warning(
9918                json!({"routing/v1": {}, "broken/v1": value}),
9919                "role_declarations values must be objects",
9920            );
9921        }
9922        assert_hello_drops_role_declarations_with_warning(
9923            json!([]),
9924            "role_declarations must be an object",
9925        );
9926    }
9927
9928    #[test]
9929    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9930        assert_hello_drops_role_declarations_with_warning(
9931            json!({"routing/v1": {}, "bad key": {}}),
9932            "role_declarations key must use capability-name grammar",
9933        );
9934    }
9935
9936    #[test]
9937    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9938        assert_hello_drops_role_declarations_with_warning(
9939            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9940            "role_declarations exceeds 16384 compact JSON bytes",
9941        );
9942    }
9943
9944    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9945        let mut body = serde_json::to_value(ModuleHelloBody {
9946            manifest: manifest("flow-ops", PROTOCOL_VERSION),
9947            protocol_ver: PROTOCOL_VERSION,
9948            control_ops: None,
9949            launch_nonce: None,
9950        })
9951        .unwrap();
9952        body["manifest"]["provides"] = json!([{
9953            "role": "management_surface",
9954            "operations": [
9955                {"name": "diagnostics.findings", "kind": "query"},
9956                {"name": "reset", "kind": "mutate"}
9957            ],
9958            "config_schema": {}, "observability": [], "identity_scope": []
9959        }]);
9960        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9961        if let Some(flow_operations) = flow_operations {
9962            body["manifest"]["flow_operations"] = flow_operations;
9963        }
9964        Frame::build(
9965            FrameType::Hello,
9966            control_flags(),
9967            0,
9968            0,
9969            401,
9970            serde_json::to_vec(&body).unwrap(),
9971        )
9972        .unwrap()
9973    }
9974
9975    #[test]
9976    fn catalog_list_mirrors_flow_operations_unchanged() {
9977        // Order is kept as declared (not sorted) and an empty list stays an
9978        // affirmative "none", distinct from an absent declaration.
9979        for (capability, flow_operations) in [
9980            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9981            ("call-modes/v1", json!(["diagnostics.findings"])),
9982            ("flow-scopes/v1", json!([])),
9983        ] {
9984            let registry = Arc::new(Registry::default());
9985            let handler = ControlHandler::new(Arc::clone(&registry));
9986            let connection = ConnectionId::new(401);
9987            let response = handler
9988                .handle_control(
9989                    connection,
9990                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9991                )
9992                .unwrap();
9993            assert_eq!(response[0].header.ty, FrameType::HelloAck);
9994            let registered = registry
9995                .get_module_by_connection(connection)
9996                .unwrap()
9997                .unwrap()
9998                .manifest;
9999            assert_eq!(
10000                serde_json::to_value(&registered.flow_operations).unwrap(),
10001                flow_operations
10002            );
10003            let (raw, entry) = role_declarations_catalog(&handler);
10004            assert_eq!(raw["flow_operations"], flow_operations);
10005            assert_eq!(entry.flow_operations, registered.flow_operations);
10006        }
10007
10008        let registry = Arc::new(Registry::default());
10009        let handler = ControlHandler::new(Arc::clone(&registry));
10010        let response = handler
10011            .handle_control(
10012                ConnectionId::new(401),
10013                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10014            )
10015            .unwrap();
10016        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10017        let (raw, entry) = role_declarations_catalog(&handler);
10018        assert!(raw.get("flow_operations").is_none());
10019        assert_eq!(entry.flow_operations, None);
10020    }
10021
10022    fn assert_hello_drops_flow_operations_with_warning(
10023        capabilities: Value,
10024        flow_operations: Value,
10025        reason: &str,
10026    ) {
10027        let registry = Arc::new(Registry::default());
10028        let handler = ControlHandler::new(Arc::clone(&registry));
10029        let capture = EventCapture::default();
10030        let _subscriber =
10031            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10032        let response = handler
10033            .handle_control(
10034                ConnectionId::new(401),
10035                hello_with_flow_operations(capabilities, Some(flow_operations)),
10036            )
10037            .unwrap();
10038        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10039        let registered = registry
10040            .get_module_by_connection(ConnectionId::new(401))
10041            .unwrap()
10042            .expect("the module still registers")
10043            .manifest;
10044        assert_eq!(registered.module_id, "flow-ops");
10045        assert_eq!(registered.flow_operations, None);
10046        let (raw, entry) = role_declarations_catalog(&handler);
10047        assert_eq!(raw["module_id"], "flow-ops");
10048        assert!(raw.get("flow_operations").is_none());
10049        assert_eq!(entry.flow_operations, None);
10050        let warnings: Vec<_> = capture
10051            .events()
10052            .into_iter()
10053            .filter(|event| {
10054                event.fields.get("message").map(String::as_str)
10055                    == Some("dropping flow_operations from HELLO")
10056            })
10057            .collect();
10058        assert_eq!(warnings.len(), 1);
10059        assert_eq!(warnings[0].level, tracing::Level::WARN);
10060        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10061        assert_eq!(warnings[0].fields["reason"], reason);
10062    }
10063
10064    #[test]
10065    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10066        assert_hello_drops_flow_operations_with_warning(
10067            json!(["flow-scopes/v1"]),
10068            json!(["diagnostics.findings", "missing"]),
10069            "flow_operations names 'missing', which is not a declared management operation",
10070        );
10071    }
10072
10073    #[test]
10074    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10075        assert_hello_drops_flow_operations_with_warning(
10076            json!([]),
10077            json!(["diagnostics.findings"]),
10078            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10079        );
10080        // A run-only capability does not admit flows either.
10081        assert_hello_drops_flow_operations_with_warning(
10082            json!(["agent-run-scopes/v1"]),
10083            json!([]),
10084            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10085        );
10086    }
10087
10088    #[test]
10089    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10090        assert_hello_drops_flow_operations_with_warning(
10091            json!(["flow-scopes/v1"]),
10092            json!(["reset", "reset"]),
10093            "flow_operations names 'reset' more than once",
10094        );
10095        assert_hello_drops_flow_operations_with_warning(
10096            json!(["flow-scopes/v1"]),
10097            json!(vec!["reset"; 257]),
10098            "flow_operations lists 257 names, more than 256",
10099        );
10100        assert_hello_drops_flow_operations_with_warning(
10101            json!(["flow-scopes/v1"]),
10102            json!("reset"),
10103            "flow_operations must be an array of operation names",
10104        );
10105    }
10106
10107    #[test]
10108    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10109        let registry = Arc::new(Registry::default());
10110        let handler = ControlHandler::new(registry.clone());
10111        let descriptor = json!({
10112            "schema": "ckios.richui.manifest/1",
10113            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10114            "bytes": 128,
10115            "chunks": 2
10116        });
10117        let malformed = [
10118            ("schema", json!("")),
10119            ("schema", json!("a".repeat(65))),
10120            ("schema", json!("Bad schema")),
10121            (
10122                "manifest_digest",
10123                json!(format!("sha256:{}", "A".repeat(64))),
10124            ),
10125            ("manifest_digest", json!("sha256:abc")),
10126            ("bytes", json!(0)),
10127            ("bytes", json!(16_777_217)),
10128            ("bytes", json!("128")),
10129            ("chunks", json!(0)),
10130            ("chunks", json!(1025)),
10131            ("chunks", json!("2")),
10132        ];
10133        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10134            let mut invalid = descriptor.clone();
10135            invalid[field] = invalid_value;
10136            let response = handler
10137                .handle_control(
10138                    ConnectionId::new(210 + index as u64),
10139                    hello_frame_with_rich_ui(
10140                        &format!("bad-rich-ui-{index}"),
10141                        PROTOCOL_VERSION,
10142                        210 + index as u64,
10143                        None,
10144                        invalid,
10145                    ),
10146                )
10147                .unwrap();
10148            assert_eq!(response[0].header.ty, FrameType::Error);
10149            let error = parse_error(&response[0]);
10150            assert_eq!(error["code"], "invalid_manifest");
10151            assert!(error["message"]
10152                .as_str()
10153                .unwrap()
10154                .contains(&format!("rich_ui.{field}")));
10155            assert!(registry
10156                .get_module(&format!("bad-rich-ui-{index}"))
10157                .unwrap()
10158                .is_none());
10159        }
10160    }
10161
10162    #[test]
10163    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10164        let registry = Arc::new(Registry::default());
10165        let handler = ControlHandler::new(Arc::clone(&registry));
10166        let connection = ConnectionId::new(219);
10167        registry
10168            .register_with_control_ops(
10169                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10170                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10171                        "bad schema",
10172                        format!("sha256:{}", "a".repeat(64)),
10173                        128,
10174                        2,
10175                    )))
10176                    .build(),
10177                PROTOCOL_VERSION,
10178                connection,
10179                Vec::new(),
10180            )
10181            .unwrap();
10182        let request =
10183            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10184        let response = handler
10185            .handle_catalog_update(connection, request, Vec::new(), None, None)
10186            .unwrap();
10187        assert_eq!(response[0].header.ty, FrameType::Error);
10188        let error = parse_error(&response[0]);
10189        assert_eq!(error["code"], "invalid_manifest");
10190        assert!(error["message"]
10191            .as_str()
10192            .unwrap()
10193            .contains("rich_ui.schema"));
10194    }
10195
10196    #[test]
10197    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10198        let registry = Arc::new(Registry::default());
10199        let supervisor = SupervisorHandle::new();
10200        supervisor.open_swap("provider", "candidate-nonce".to_string());
10201        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10202        let connection = ConnectionId::new(218);
10203        let hello = handler
10204            .handle_control(
10205                connection,
10206                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10207            )
10208            .unwrap();
10209        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10210        assert!(registry.list_modules().unwrap().1.is_empty());
10211
10212        registry.promote_candidate("provider").unwrap().unwrap();
10213        let request = Frame::build(
10214            FrameType::Request,
10215            control_flags(),
10216            0,
10217            0,
10218            219,
10219            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10220        )
10221        .unwrap();
10222        let response = handler.handle_catalog_list(request, None).unwrap();
10223        let ClientControlResponse::CatalogList { modules, .. } =
10224            serde_json::from_slice(&response[0].body).unwrap()
10225        else {
10226            panic!("expected catalog.list");
10227        };
10228        assert_eq!(modules.len(), 1);
10229        assert!(modules[0].nonce_verified);
10230    }
10231
10232    #[test]
10233    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10234        let registry = Arc::new(Registry::default());
10235        let handler = ControlHandler::new(Arc::clone(&registry));
10236        let hello = Frame::build(
10237            FrameType::Hello,
10238            control_flags(),
10239            0,
10240            0,
10241            220,
10242            serde_json::to_vec(&ModuleHelloBody {
10243                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10244                protocol_ver: PROTOCOL_VERSION,
10245                control_ops: None,
10246                launch_nonce: Some("unverified".to_string()),
10247            })
10248            .unwrap(),
10249        )
10250        .unwrap();
10251        assert_eq!(
10252            handler
10253                .handle_control(ConnectionId::new(220), hello)
10254                .unwrap()[0]
10255                .header
10256                .ty,
10257            FrameType::HelloAck
10258        );
10259        let request = Frame::build(
10260            FrameType::Request,
10261            control_flags(),
10262            0,
10263            0,
10264            221,
10265            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10266        )
10267        .unwrap();
10268        let response = handler.handle_catalog_list(request, None).unwrap();
10269        let ClientControlResponse::CatalogList { modules, .. } =
10270            serde_json::from_slice(&response[0].body).unwrap()
10271        else {
10272            panic!("expected catalog.list");
10273        };
10274        assert_eq!(
10275            serde_json::to_vec(&modules[0]).unwrap(),
10276            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10277        );
10278        assert!(!modules[0].nonce_verified);
10279    }
10280
10281    #[test]
10282    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10283        let registry = Arc::new(Registry::default());
10284        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10285            [("vault".to_string(), true), ("squatter".to_string(), true)],
10286            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10287        );
10288        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10289        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10290            provides: vec!["credentials-provider/v1".to_string()],
10291            requires: Vec::new(),
10292            must_never_reach: Vec::new(),
10293        });
10294        let frame = Frame::build(
10295            FrameType::Hello,
10296            control_flags(),
10297            0,
10298            0,
10299            77,
10300            serde_json::to_vec(&ModuleHelloBody {
10301                manifest: squatter,
10302                protocol_ver: PROTOCOL_VERSION,
10303                control_ops: None,
10304                launch_nonce: None,
10305            })
10306            .expect("HELLO serializes"),
10307        )
10308        .expect("HELLO frame builds");
10309        let response = handler
10310            .handle_control(ConnectionId::new(77), frame)
10311            .expect("reserved claim receives a typed refusal");
10312        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10313        assert_eq!(
10314            registry
10315                .active_registration_count()
10316                .expect("registry reads"),
10317            0,
10318            "a reserved capability refusal must not leave a catalog entry"
10319        );
10320    }
10321
10322    #[test]
10323    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10324        for settlement in ["timeout", "inconclusive", "drop"] {
10325            let breakers = RouteBindBreakers::default();
10326            let RouteBindAdmission::Admitted {
10327                guard: mut old,
10328                probe: false,
10329            } = breakers.admit("prov")
10330            else {
10331                panic!("ordinary relay admitted")
10332            };
10333            let RouteBindAdmission::Admitted {
10334                guard: mut opener, ..
10335            } = breakers.admit("prov")
10336            else {
10337                panic!("second relay admitted")
10338            };
10339            assert!(
10340                !opener
10341                    .record_timeout(1, Duration::ZERO)
10342                    .unwrap()
10343                    .reopened_after_probe
10344            );
10345            let RouteBindAdmission::Admitted {
10346                guard: mut probe,
10347                probe: true,
10348            } = breakers.admit("prov")
10349            else {
10350                panic!("one cooldown probe admitted")
10351            };
10352            match settlement {
10353                "timeout" => assert!(
10354                    !old.record_timeout(1, Duration::ZERO)
10355                        .unwrap()
10356                        .reopened_after_probe
10357                ),
10358                "inconclusive" => old.record_inconclusive(),
10359                "drop" => drop(old),
10360                _ => unreachable!(),
10361            }
10362            assert!(
10363                matches!(
10364                    breakers.admit("prov"),
10365                    RouteBindAdmission::Refused {
10366                        probe_in_flight: true,
10367                        ..
10368                    }
10369                ),
10370                "{settlement} of a pre-open relay cannot release the real probe"
10371            );
10372            assert!(
10373                probe
10374                    .record_timeout(1, Duration::ZERO)
10375                    .unwrap()
10376                    .reopened_after_probe
10377            );
10378            assert!(matches!(
10379                breakers.admit("prov"),
10380                RouteBindAdmission::Admitted { probe: true, .. }
10381            ));
10382        }
10383        let breakers = RouteBindBreakers::default();
10384        let admit = || match breakers.admit("prov") {
10385            RouteBindAdmission::Admitted { guard, .. } => guard,
10386            _ => panic!("relay admitted"),
10387        };
10388        admit().record_timeout(1, Duration::ZERO);
10389        let mut old_probe = admit();
10390        breakers.reset_for_new_module_connection("prov");
10391        admit().record_timeout(1, Duration::ZERO);
10392        let _new_probe = admit();
10393        old_probe.record_inconclusive();
10394        assert!(matches!(
10395            breakers.admit("prov"),
10396            RouteBindAdmission::Refused {
10397                probe_in_flight: true,
10398                ..
10399            }
10400        ));
10401    }
10402
10403    #[tokio::test]
10404    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10405        for candidate in [false, true] {
10406            let registry = Arc::new(Registry::default());
10407            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10408                [("vault".to_string(), true), ("squatter".to_string(), true)],
10409                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10410            );
10411            let conn = ConnectionId::new(77);
10412            let (ctx, mut rx) = route_ctx(conn);
10413            let initial = capability_manifest("squatter", &[], &[]);
10414            if candidate {
10415                registry
10416                    .register_candidate_with_control_ops(
10417                        initial.clone(),
10418                        PROTOCOL_VERSION,
10419                        conn,
10420                        module_baseline_control_ops(),
10421                    )
10422                    .unwrap();
10423                handler
10424                    .forwarding
10425                    .register_candidate_module_connection(
10426                        conn,
10427                        "squatter".to_string(),
10428                        PROTOCOL_VERSION,
10429                        manifest_concurrency(&initial),
10430                        ctx.egress.clone(),
10431                    )
10432                    .unwrap();
10433            } else {
10434                hello_via_sink(
10435                    &handler,
10436                    &ctx,
10437                    &mut rx,
10438                    hello_frame_with_manifest(initial.clone(), 1),
10439                )
10440                .await;
10441            }
10442            let response = handler
10443                .handle_control_frame(
10444                    &ctx,
10445                    catalog_update_with_capabilities_frame(
10446                        2,
10447                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
10448                            .capabilities
10449                            .unwrap(),
10450                    ),
10451                )
10452                .await
10453                .unwrap();
10454            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10455            assert_eq!(
10456                registry
10457                    .get_module_by_connection(conn)
10458                    .unwrap()
10459                    .unwrap()
10460                    .manifest,
10461                initial
10462            );
10463        }
10464    }
10465
10466    #[test]
10467    fn server_describe_surfaces_required_capability_verdict_fields() {
10468        let registry = Arc::new(Registry::default());
10469        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10470            [
10471                ("consumer".to_string(), true),
10472                ("provider".to_string(), false),
10473            ],
10474            BTreeMap::new(),
10475        );
10476        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10477        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10478            provides: Vec::new(),
10479            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10480                capability: "credentials-provider/v1".to_string(),
10481                need: subc_protocol::manifest::CapabilityNeed::Required,
10482            }],
10483            must_never_reach: Vec::new(),
10484        });
10485        let hello = Frame::build(
10486            FrameType::Hello,
10487            control_flags(),
10488            0,
10489            0,
10490            78,
10491            serde_json::to_vec(&ModuleHelloBody {
10492                manifest: consumer,
10493                protocol_ver: PROTOCOL_VERSION,
10494                control_ops: None,
10495                launch_nonce: None,
10496            })
10497            .expect("HELLO serializes"),
10498        )
10499        .expect("HELLO frame builds");
10500        handler
10501            .handle_control(ConnectionId::new(78), hello)
10502            .expect("consumer registers");
10503        let describe = Frame::build(
10504            FrameType::Request,
10505            control_flags(),
10506            0,
10507            0,
10508            79,
10509            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10510                .expect("request serializes"),
10511        )
10512        .expect("describe frame builds");
10513        let response = handler
10514            .handle_server_describe(describe)
10515            .expect("server.describe succeeds");
10516        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10517        let requirement = &rendered["capability_requirements"][0];
10518        assert_eq!(requirement["consumer"], "consumer");
10519        assert_eq!(requirement["verdict"], "never_provided");
10520        assert_eq!(requirement["episode_seq"], 1);
10521        assert_eq!(requirement["config_satisfiable"], false);
10522        assert_eq!(requirement["runtime_available"], false);
10523        assert!(requirement["detail"]
10524            .as_str()
10525            .expect("detail string")
10526            .contains("credentials-provider/v1"));
10527    }
10528
10529    /// Register a consumer that requires `project-identity/v1` and the provider
10530    /// of it, optionally mark daemon shutdown as begun, then deregister the
10531    /// provider as its connection closes. Returns the `capability.requirement`
10532    /// events that deregistration logged and the verdict the evaluator then
10533    /// holds for the requirement (the one `route.open` refusals read).
10534    fn deregister_required_provider(
10535        daemon_shutdown_begun: bool,
10536    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10537        const CAPABILITY: &str = "project-identity/v1";
10538        let registry = Arc::new(Registry::default());
10539        let daemon_shutdown = DaemonShutdownFlag::default();
10540        let handler = ControlHandler::new(Arc::clone(&registry))
10541            .with_capability_config(
10542                [
10543                    ("consumer".to_string(), true),
10544                    ("provider".to_string(), true),
10545                ],
10546                BTreeMap::new(),
10547            )
10548            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10549        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10550        consumer.capabilities = Some(CapabilityDeclarations {
10551            provides: Vec::new(),
10552            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10553                capability: CAPABILITY.to_string(),
10554                need: subc_protocol::manifest::CapabilityNeed::Required,
10555            }],
10556            must_never_reach: Vec::new(),
10557        });
10558        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10559        let provider_conn = ConnectionId::new(182);
10560        for (conn, manifest, corr) in [
10561            (ConnectionId::new(181), consumer, 181),
10562            (provider_conn, provider, 182),
10563        ] {
10564            let hello = handler
10565                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10566                .expect("module registers");
10567            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10568        }
10569        assert_eq!(
10570            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10571            Some(CapabilityVerdict::Provided),
10572            "the registered provider must satisfy the requirement first"
10573        );
10574        if daemon_shutdown_begun {
10575            daemon_shutdown.set_for_test();
10576        }
10577
10578        let capture = EventCapture::default();
10579        let _subscriber =
10580            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10581        handler
10582            .cleanup_connection(provider_conn)
10583            .expect("provider deregisters");
10584        let events = capture
10585            .events()
10586            .into_iter()
10587            .filter(|event| {
10588                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10589            })
10590            .collect();
10591        (
10592            events,
10593            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10594        )
10595    }
10596
10597    #[test]
10598    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10599        let (events, verdict) = deregister_required_provider(true);
10600        assert_eq!(
10601            verdict,
10602            Some(CapabilityVerdict::NeverProvided),
10603            "shutdown must not change the verdict route.open refusals read"
10604        );
10605        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10606        let event = &events[0];
10607        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10608        assert_eq!(
10609            event.fields.get("cause").map(String::as_str),
10610            Some("\"daemon_shutdown\""),
10611            "{event:?}"
10612        );
10613        assert_eq!(
10614            event.fields.get("verdict").map(String::as_str),
10615            Some("\"never_provided\""),
10616            "{event:?}"
10617        );
10618    }
10619
10620    #[test]
10621    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10622        let (events, verdict) = deregister_required_provider(false);
10623        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10624        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10625        let event = &events[0];
10626        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10627        assert_eq!(event.fields.get("cause"), None, "{event:?}");
10628        assert_eq!(
10629            event.fields.get("verdict").map(String::as_str),
10630            Some("\"never_provided\""),
10631            "{event:?}"
10632        );
10633    }
10634
10635    #[test]
10636    fn catalog_list_omits_capabilities_for_legacy_manifest() {
10637        let registry = Arc::new(Registry::default());
10638        let handler = ControlHandler::new(Arc::clone(&registry));
10639        let hello = handler
10640            .handle_control(
10641                ConnectionId::new(101),
10642                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10643            )
10644            .expect("legacy HELLO registers");
10645        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10646
10647        let request = Frame::build(
10648            FrameType::Request,
10649            control_flags(),
10650            0,
10651            0,
10652            102,
10653            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10654                .expect("catalog request serializes"),
10655        )
10656        .expect("catalog request frame builds");
10657        let response = handler
10658            .handle_catalog_list(request, None)
10659            .expect("catalog list succeeds");
10660        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10661        assert!(
10662            body["modules"][0].get("capabilities").is_none(),
10663            "legacy manifest must retain an absent capabilities field on catalog.list"
10664        );
10665    }
10666
10667    #[test]
10668    fn hello_ack_omits_storage_when_no_storage_config() {
10669        let registry = Arc::new(Registry::default());
10670        let handler = ControlHandler::new(Arc::clone(&registry));
10671        let responses = handler
10672            .handle_control(
10673                ConnectionId::new(1),
10674                hello_frame("aft", PROTOCOL_VERSION, 7),
10675            )
10676            .unwrap();
10677        let ack = parse_ack(&responses[0]);
10678        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10679        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10680    }
10681
10682    #[tokio::test]
10683    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10684        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10685        let registry = Arc::new(Registry::default());
10686        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
10687        let responses = handler
10688            .handle_control(
10689                ConnectionId::new(1),
10690                hello_frame("aft", PROTOCOL_VERSION, 7),
10691            )
10692            .unwrap();
10693        let ack = parse_ack(&responses[0]);
10694        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10695
10696        let described = handler
10697            .handle_control_frame(
10698                &route_ctx(ConnectionId::new(2)).0,
10699                Frame::build(
10700                    FrameType::Request,
10701                    control_flags(),
10702                    0,
10703                    0,
10704                    9,
10705                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10706                )
10707                .unwrap(),
10708            )
10709            .await
10710            .unwrap();
10711        let ClientControlResponse::ServerDescribe { machine_id, .. } =
10712            serde_json::from_slice(&described[0].body).unwrap()
10713        else {
10714            panic!("server.describe answered with another shape");
10715        };
10716        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10717    }
10718
10719    #[test]
10720    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10721        // With a central sqlite storage policy, each registering module gets its
10722        // own resolved descriptor in HELLO_ACK, keyed by its module id.
10723        let registry = Arc::new(Registry::default());
10724        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
10725            crate::daemon_config::StorageConfig::Sqlite {
10726                data_home: std::path::PathBuf::from("/data"),
10727            },
10728        ));
10729
10730        let responses = handler
10731            .handle_control(
10732                ConnectionId::new(1),
10733                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10734            )
10735            .unwrap();
10736        let ack = parse_ack(&responses[0]);
10737        assert_eq!(
10738            ack.storage,
10739            Some(serde_json::json!({
10740                "module_id": "alfonso-routing",
10741                "storage_namespace": "default",
10742                "isolation": { "kind": "module" },
10743                "backend": {
10744                    "backend": "sqlite",
10745                    "path": "/data/cortexkit/alfonso-routing/store.db"
10746                }
10747            })),
10748            "the delivered descriptor is the module's own sqlite store path"
10749        );
10750    }
10751
10752    #[test]
10753    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10754        let registry = Arc::new(Registry::default());
10755        let handler = ControlHandler::new(Arc::clone(&registry));
10756        let conn = ConnectionId::new(1);
10757        let responses = handler
10758            .handle_control(
10759                conn,
10760                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10761            )
10762            .unwrap();
10763        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10764        let registration = registry.get_module("aft").unwrap().unwrap();
10765        assert_eq!(registration.control_ops, module_baseline_control_ops());
10766
10767        let frame =
10768            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10769        assert!(handler
10770            .guard_module_control_op(&frame, "aft", "route.bind")
10771            .unwrap()
10772            .is_none());
10773        let error = handler
10774            .guard_module_control_op(&frame, "aft", "test.synthetic")
10775            .unwrap()
10776            .expect("synthetic ungranted op should be rejected");
10777        assert_eq!(error.header.ty, FrameType::Error);
10778        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10779    }
10780
10781    #[test]
10782    fn hello_control_ops_some_adds_optional_grants() {
10783        let registry = Arc::new(Registry::default());
10784        let handler = ControlHandler::new(Arc::clone(&registry));
10785        handler
10786            .handle_control(
10787                ConnectionId::new(1),
10788                hello_frame_with_control_ops(
10789                    "aft",
10790                    PROTOCOL_VERSION,
10791                    7,
10792                    Some(vec![
10793                        "future.synthetic".to_string(),
10794                        "route.bind".to_string(),
10795                    ]),
10796                ),
10797            )
10798            .unwrap();
10799        let registration = registry.get_module("aft").unwrap().unwrap();
10800        assert_eq!(
10801            registration.control_ops,
10802            vec![
10803                "route.bind".to_string(),
10804                "route.status".to_string(),
10805                "future.synthetic".to_string(),
10806            ]
10807        );
10808        let frame =
10809            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10810        assert!(handler
10811            .guard_module_control_op(&frame, "aft", "future.synthetic")
10812            .unwrap()
10813            .is_none());
10814    }
10815
10816    #[tokio::test]
10817    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10818        let registry = Arc::new(Registry::default());
10819        let forwarding = Arc::new(ForwardingTable::default());
10820        let handler =
10821            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10822        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10823        hello_via_sink(
10824            &handler,
10825            &module_ctx,
10826            &mut module_rx,
10827            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10828        )
10829        .await;
10830
10831        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10832        let responses = handler
10833            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10834            .await
10835            .unwrap();
10836        assert_eq!(responses.len(), 1);
10837        assert_eq!(responses[0].header.ty, FrameType::Error);
10838        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10839        assert!(module_rx.try_recv().is_err());
10840    }
10841
10842    #[tokio::test]
10843    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10844        let registry = Arc::new(Registry::default());
10845        let forwarding = Arc::new(ForwardingTable::default());
10846        let handler =
10847            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10848        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10849        hello_via_sink(
10850            &handler,
10851            &module_ctx,
10852            &mut module_rx,
10853            hello_frame_with_control_ops(
10854                "aft",
10855                PROTOCOL_VERSION,
10856                7,
10857                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10858            ),
10859        )
10860        .await;
10861
10862        let project_root = unique_project_root("demux");
10863        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10864        let route_handler = handler.clone();
10865        let route_task = tokio::spawn(async move {
10866            route_handler
10867                .handle_control_frame(
10868                    &route_client_ctx,
10869                    route_open_frame(100, "aft", project_root),
10870                )
10871                .await
10872                .unwrap()
10873        });
10874        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10875            .await
10876            .unwrap()
10877            .unwrap();
10878        assert!(matches!(
10879            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10880            ModuleControlRequest::RouteBind { .. }
10881        ));
10882
10883        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10884        let health_handler = handler.clone();
10885        let health_task = tokio::spawn(async move {
10886            health_handler
10887                .handle_control_frame(
10888                    &health_client_ctx,
10889                    supervisor_health_probe_frame(101, "aft"),
10890                )
10891                .await
10892                .unwrap()
10893        });
10894        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10895            .await
10896            .unwrap()
10897            .unwrap();
10898        assert_eq!(
10899            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10900            ModuleControlRequest::HealthCheck {}
10901        );
10902
10903        handler
10904            .handle_control_frame(
10905                &module_ctx,
10906                health_response(health_frame.header.corr, HealthStatus::Degraded),
10907            )
10908            .await
10909            .unwrap();
10910        let health_response = health_task.await.unwrap();
10911        assert_eq!(health_response.len(), 1);
10912        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10913            ClientControlResponse::SupervisorHealthProbe {
10914                module_id,
10915                status,
10916                detail,
10917                metrics,
10918            } => {
10919                assert_eq!(module_id, "aft");
10920                assert_eq!(status, HealthStatus::Degraded);
10921                assert_eq!(detail.as_deref(), Some("warming"));
10922                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10923            }
10924            other => panic!("unexpected health response: {other:?}"),
10925        }
10926
10927        handler
10928            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10929            .await
10930            .unwrap();
10931        let route_response = route_task.await.unwrap();
10932        assert!(route_response.is_empty());
10933        let published = route_client_rx.recv().await.unwrap();
10934        assert!(matches!(
10935            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10936            ClientControlResponse::RouteOpen { .. }
10937        ));
10938    }
10939
10940    /// Start one `route.open` on `client_connection` and return its still-running
10941    /// handler task together with the `route.bind` the module received for it.
10942    /// The handler blocks until the module answers, so it has to run as a task
10943    /// while the test drives the module side.
10944    async fn relay_route_open(
10945        handler: &ControlHandler,
10946        client_connection: ConnectionId,
10947        client_egress: &FrameSink,
10948        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10949        corr: u64,
10950        module_id: &str,
10951        project_root_label: &str,
10952    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10953        let ctx = RouteCtx {
10954            connection_id: client_connection,
10955            egress: client_egress.clone(),
10956        };
10957        let handler = handler.clone();
10958        let project_root = unique_project_root(project_root_label);
10959        let module_id = module_id.to_string();
10960        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10961        let task = tokio::spawn(async move {
10962            let _guard = tracing::dispatcher::set_default(&dispatch);
10963            handler
10964                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10965                .await
10966                .unwrap()
10967        });
10968        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10969            .await
10970            .expect("module receives the relayed route.bind")
10971            .expect("module egress is open");
10972        (task, bind.frame)
10973    }
10974
10975    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10976        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10977            ModuleControlRequest::RouteBind {
10978                route_channel,
10979                epoch,
10980                ..
10981            } => (route_channel, epoch),
10982            other => panic!("expected a route.bind request, got {other:?}"),
10983        }
10984    }
10985
10986    fn published_route(frame: &Frame) -> (u16, u32) {
10987        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10988            ClientControlResponse::RouteOpen {
10989                route_channel,
10990                route_epoch,
10991            } => (route_channel, route_epoch),
10992            other => panic!("expected a route.open response, got {other:?}"),
10993        }
10994    }
10995
10996    /// Reproduction of a production outage. A client had `route.open`s in
10997    /// flight to a module and was already marked closing -- its egress had refused a
10998    /// module frame, so the daemon asked its connection to end -- while its sink
10999    /// was still open. When the module acked those binds, the daemon refused to
11000    /// commit a route for a closing client, and that refusal was returned from
11001    /// the MODULE connection's frame handler, where a router error that has no
11002    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
11003    /// the supervisor correctly did not respawn a clean exit, and every seat lost
11004    /// its tools for hours -- one client's teardown took down a connection
11005    /// carrying ~170 other routes.
11006    ///
11007    /// The window is opened here by calling the production path that opens it
11008    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
11009    /// state that matters is "in `closing_connections`, sink still open, relay
11010    /// still pending", and it lasts only from the close request until the
11011    /// connection loop reacts to it; a socket-level test can flood a client into
11012    /// that escalation but cannot pin the module's ack inside the window. Closing
11013    /// the socket instead takes the other path entirely -- connection teardown
11014    /// removes the pending relay under the same lock, so the ack finds nothing.
11015    #[tokio::test]
11016    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11017        let registry = Arc::new(Registry::default());
11018        let forwarding = Arc::new(ForwardingTable::default());
11019        let handler =
11020            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11021
11022        let module_connection = ConnectionId::new(30);
11023        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11024        hello_via_sink(
11025            &handler,
11026            &module_ctx,
11027            &mut module_rx,
11028            hello_frame("aft", PROTOCOL_VERSION, 7),
11029        )
11030        .await;
11031
11032        let dying_client = ConnectionId::new(31);
11033        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11034
11035        // A published route on the dying client. The escalation below only marks
11036        // a connection closing for a route it has already published.
11037        let (first_task, first_bind) = relay_route_open(
11038            &handler,
11039            dying_client,
11040            &dying_ctx.egress,
11041            &mut module_rx,
11042            100,
11043            "aft",
11044            "closing-first",
11045        )
11046        .await;
11047        handler
11048            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11049            .await
11050            .unwrap();
11051        assert!(first_task.await.unwrap().is_empty());
11052        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11053
11054        // A second route.open from the same client, relayed and awaiting its ack.
11055        let (second_task, second_bind) = relay_route_open(
11056            &handler,
11057            dying_client,
11058            &dying_ctx.egress,
11059            &mut module_rx,
11060            101,
11061            "aft",
11062            "closing-second",
11063        )
11064        .await;
11065        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11066
11067        // The window: the client is closing, its sink is still open, and its
11068        // second bind is still pending.
11069        assert!(forwarding
11070            .escalate_client_delivery_failure(
11071                dying_client,
11072                first_channel,
11073                first_epoch,
11074                CloseReason::new(
11075                    "module_to_client_delivery_failed",
11076                    "client egress refused a module frame",
11077                ),
11078                crate::forwarding::UndeliveredFrame {
11079                    module_id: None,
11080                    sink: &dying_ctx.egress,
11081                },
11082            )
11083            .unwrap());
11084        assert!(!dying_ctx.egress.is_closed());
11085
11086        // The frame that used to end the module connection.
11087        let ack = handler
11088            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11089            .await;
11090        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11091        if module_loop_error.is_some() {
11092            // What the server's connection loop does with a router error that has
11093            // no ERROR-frame translation: end the connection, which releases the
11094            // module's registration and every route on it.
11095            handler.cleanup_connection(module_connection).unwrap();
11096        }
11097        // Read the module's next frame before opening the co-tenant's route, so
11098        // the GOODBYE assertion below is about THIS ack and not about later
11099        // traffic. `None` means the module was told nothing.
11100        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11101            .await
11102            .ok()
11103            .flatten();
11104
11105        // 1. The module connection is still registered.
11106        assert!(
11107            registry
11108                .get_module_by_connection(module_connection)
11109                .unwrap()
11110                .is_some(),
11111            "one client's closing connection ended the shared module connection: \
11112             {module_loop_error:?}"
11113        );
11114        // ...and still serving: another client can open and use a route on it.
11115        let cotenant = ConnectionId::new(32);
11116        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11117        let (cotenant_task, cotenant_bind) = relay_route_open(
11118            &handler,
11119            cotenant,
11120            &cotenant_ctx.egress,
11121            &mut module_rx,
11122            102,
11123            "aft",
11124            "closing-cotenant",
11125        )
11126        .await;
11127        handler
11128            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11129            .await
11130            .unwrap();
11131        assert!(cotenant_task.await.unwrap().is_empty());
11132        let (cotenant_channel, cotenant_epoch) =
11133            published_route(&cotenant_rx.recv().await.unwrap());
11134        assert!(matches!(
11135            forwarding
11136                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11137                .unwrap(),
11138            DataRoute::Client(DataRouteState::Bound(_))
11139        ));
11140
11141        // 2. The module was told to drop the binding it created for the route
11142        //    that will never be published.
11143        let goodbye = post_ack_module_frame
11144            .expect("module receives a GOODBYE for the abandoned route channel");
11145        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11146        assert_eq!(goodbye.header.channel, abandoned_channel);
11147        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11148
11149        // 3. The dying client received nothing: no route was ever published to
11150        //    it. Its route.open is answered as unavailable, which the connection
11151        //    loop would write to a socket that is already going away.
11152        assert!(dying_rx.try_recv().is_err());
11153        let second_response = second_task.await.unwrap();
11154        assert_eq!(second_response.len(), 1);
11155        assert_eq!(
11156            parse_error(&second_response[0])["code"],
11157            "target_unavailable"
11158        );
11159    }
11160
11161    /// One `catalog.list` as a client sees it: the generation and the listed
11162    /// module ids with their readiness.
11163    async fn catalog_snapshot(
11164        handler: &ControlHandler,
11165        client_ctx: &RouteCtx,
11166        corr: u64,
11167    ) -> (u64, Vec<(String, bool)>) {
11168        let body =
11169            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11170        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11171        let reply = handler
11172            .handle_control_frame(client_ctx, request)
11173            .await
11174            .unwrap()
11175            .pop()
11176            .unwrap();
11177        let ClientControlResponse::CatalogList {
11178            generation,
11179            modules,
11180            ..
11181        } = serde_json::from_slice(&reply.body).unwrap()
11182        else {
11183            panic!("not a catalog.list reply");
11184        };
11185        (
11186            generation,
11187            modules
11188                .into_iter()
11189                .map(|entry| (entry.module_id, entry.ready))
11190                .collect(),
11191        )
11192    }
11193
11194    /// A handler whose HELLO for module "aft" on connection 40 is parked
11195    /// between reserving its registry entry and installing its forwarding
11196    /// endpoint. Returns the handler, the two-party barrier (wait once to know
11197    /// the HELLO is parked, once more to release it), the HELLO's thread, and
11198    /// the module's context and receiver.
11199    fn hello_held_before_forwarding() -> (
11200        ControlHandler,
11201        Arc<std::sync::Barrier>,
11202        std::thread::JoinHandle<Vec<Frame>>,
11203        RouteCtx,
11204        mpsc::Receiver<crate::router::OutboundFrame>,
11205    ) {
11206        let hold = Arc::new(std::sync::Barrier::new(2));
11207        let handler = ControlHandler::with_forwarding(
11208            Arc::new(Registry::default()),
11209            Arc::new(ForwardingTable::default()),
11210        )
11211        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11212        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11213        let hello = {
11214            let handler = handler.clone();
11215            let module_ctx = module_ctx.clone();
11216            std::thread::spawn(move || {
11217                handler
11218                    .handle_hello(
11219                        module_ctx.connection_id,
11220                        Some(module_ctx.egress.clone()),
11221                        hello_frame("aft", PROTOCOL_VERSION, 7),
11222                    )
11223                    .unwrap()
11224            })
11225        };
11226        (handler, hold, hello, module_ctx, module_rx)
11227    }
11228
11229    /// While a HELLO has reserved its registry entry but its forwarding
11230    /// endpoint is not installed yet, `route.open` to the module can only be
11231    /// refused. `catalog.list` must not list the module in that window: a
11232    /// client that opens a route as soon as it sees a module listed and ready
11233    /// would be refused for a module the daemon had just advertised. The
11234    /// refusal in the window stays the retryable `target_unavailable`, never
11235    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
11236    /// Once the HELLO completes, the module is listed and a route to it opens.
11237    #[tokio::test]
11238    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11239        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11240        let client = ConnectionId::new(41);
11241        let (client_ctx, mut client_rx) = route_ctx(client);
11242
11243        hold.wait();
11244        // Parked: the registry holds "aft", forwarding has no endpoint for it.
11245        assert!(handler.registry.get_module("aft").unwrap().is_some());
11246        assert!(!handler
11247            .forwarding
11248            .has_live_module_connection("aft")
11249            .unwrap());
11250        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11251        assert!(
11252            listed.is_empty(),
11253            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11254        );
11255        let refused = handler
11256            .handle_control_frame(
11257                &client_ctx,
11258                route_open_frame(301, "aft", unique_project_root("listed-window")),
11259            )
11260            .await
11261            .unwrap();
11262        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11263        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11264        let duplicate = handler
11265            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11266            .await
11267            .unwrap();
11268        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11269
11270        hold.wait();
11271        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11272        let ack = module_rx.try_recv().unwrap().frame;
11273        assert_eq!(ack.header.ty, FrameType::HelloAck);
11274        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11275        assert_eq!(listed, vec![("aft".to_string(), true)]);
11276
11277        let (open_task, bind) = relay_route_open(
11278            &handler,
11279            client,
11280            &client_ctx.egress,
11281            &mut module_rx,
11282            303,
11283            "aft",
11284            "listed-after",
11285        )
11286        .await;
11287        assert!(handler
11288            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11289            .await
11290            .unwrap()
11291            .is_empty());
11292        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11293        published_route(&client_rx.try_recv().unwrap().frame);
11294    }
11295
11296    /// A catalog watcher keys on the generation: when it changes, it re-reads
11297    /// the listing. So the generation must advance exactly when the module
11298    /// appears. If it advanced while the module was still hidden, a watcher
11299    /// would read the new generation without the module, and the module would
11300    /// later appear under that same generation with nothing telling the
11301    /// watcher to look again.
11302    #[tokio::test]
11303    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11304        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11305        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11306        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11307
11308        hold.wait();
11309        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11310        assert!(listed.is_empty());
11311        assert_eq!(
11312            during, before,
11313            "the generation moved while the module was not listed"
11314        );
11315
11316        hold.wait();
11317        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11318        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11319        assert_eq!(listed, vec![("aft".to_string(), true)]);
11320        assert_eq!(
11321            after,
11322            before + 1,
11323            "the module appeared without exactly one generation bump"
11324        );
11325    }
11326
11327    /// A HELLO whose forwarding endpoint cannot be installed (here the
11328    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11329    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11330    /// and the catalog generation where it was.
11331    #[tokio::test]
11332    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11333        let registry = Arc::new(Registry::default());
11334        let handler = ControlHandler::with_forwarding(
11335            Arc::clone(&registry),
11336            Arc::new(ForwardingTable::default()),
11337        );
11338        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11339        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11340        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11341        drop(module_rx);
11342
11343        let replies = handler
11344            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11345            .await
11346            .unwrap();
11347        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11348        assert!(registry.get_module("aft").unwrap().is_none());
11349        assert!(registry
11350            .get_module_by_connection(module_ctx.connection_id)
11351            .unwrap()
11352            .is_none());
11353        assert_eq!(
11354            catalog_snapshot(&handler, &client_ctx, 301).await,
11355            (before, Vec::new())
11356        );
11357    }
11358
11359    /// A handler whose `route.open` parks after queueing the route.bind and
11360    /// before marking it enqueued, with module "aft" registered on connection
11361    /// 40. Returns the two-party barrier that parks and releases it.
11362    async fn route_open_held_after_bind_send() -> (
11363        ControlHandler,
11364        Arc<tokio::sync::Barrier>,
11365        RouteCtx,
11366        mpsc::Receiver<crate::router::OutboundFrame>,
11367    ) {
11368        let hold = Arc::new(tokio::sync::Barrier::new(2));
11369        let handler = ControlHandler::with_forwarding(
11370            Arc::new(Registry::default()),
11371            Arc::new(ForwardingTable::default()),
11372        )
11373        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11374        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11375        hello_via_sink(
11376            &handler,
11377            &module_ctx,
11378            &mut module_rx,
11379            hello_frame("aft", PROTOCOL_VERSION, 7),
11380        )
11381        .await;
11382        (handler, hold, module_ctx, module_rx)
11383    }
11384
11385    /// A fast module acks the route.bind, and the reader commits the route,
11386    /// before `route.open` has marked the relay enqueued. The daemon used to
11387    /// read the missing relay as an abandoned bind and send the module a
11388    /// GOODBYE for the route it had just committed: the client was told its
11389    /// route was open, the module unbound the channel, and the client's first
11390    /// request was dropped on the module side as unbound.
11391    #[tokio::test]
11392    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11393        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11394        let client = ConnectionId::new(41);
11395        let (client_ctx, mut client_rx) = route_ctx(client);
11396
11397        let (open_task, bind) = relay_route_open(
11398            &handler,
11399            client,
11400            &client_ctx.egress,
11401            &mut module_rx,
11402            200,
11403            "aft",
11404            "acked-before-mark",
11405        )
11406        .await;
11407        let (module_channel, module_epoch) = route_bind_channel(&bind);
11408        // route.open is now parked between queueing the bind and marking it.
11409        hold.wait().await;
11410
11411        // The module answers inside that window, and the route commits.
11412        assert!(handler
11413            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11414            .await
11415            .unwrap()
11416            .is_empty());
11417        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11418
11419        hold.wait().await;
11420        assert!(
11421            open_task.await.unwrap().is_empty(),
11422            "route.open answers accepted"
11423        );
11424
11425        // Every frame route.open could have sent is already queued, because it
11426        // has returned; the module must have been sent nothing.
11427        match module_rx.try_recv() {
11428            Err(mpsc::error::TryRecvError::Empty) => {}
11429            Ok(outbound) => panic!(
11430                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11431                outbound.frame.header.ty,
11432                outbound.frame.header.channel,
11433                outbound.frame.header.epoch
11434            ),
11435            Err(err) => panic!("module egress closed: {err}"),
11436        }
11437
11438        // The client's first request on the route is the next thing the module
11439        // reads, on the binding it just created.
11440        let router = Router::with_control_handler(Arc::new(handler.clone()));
11441        let request = Frame::build(
11442            FrameType::Request,
11443            Flags::new(true, Priority::Interactive, false),
11444            client_channel,
11445            client_epoch,
11446            201,
11447            b"first request".to_vec(),
11448        )
11449        .unwrap();
11450        router
11451            .route_for_connection(&client_ctx, request)
11452            .await
11453            .unwrap();
11454        let delivered = module_rx.try_recv().unwrap().frame;
11455        assert_eq!(delivered.header.ty, FrameType::Request);
11456        assert_eq!(
11457            (delivered.header.channel, delivered.header.epoch),
11458            (module_channel, module_epoch)
11459        );
11460        assert_eq!(delivered.body, b"first request");
11461    }
11462
11463    /// The client goes away while `route.open` is still queueing the bind, so
11464    /// the relay is dropped with no module reply. The teardown cannot send the
11465    /// GOODBYE itself (it could reach the module ahead of the bind), so
11466    /// `route.open` must send it once the bind is queued; otherwise the module
11467    /// keeps a binding the daemon will never commit.
11468    #[tokio::test]
11469    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11470        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11471        let client = ConnectionId::new(42);
11472        let (client_ctx, _client_rx) = route_ctx(client);
11473
11474        let (open_task, bind) = relay_route_open(
11475            &handler,
11476            client,
11477            &client_ctx.egress,
11478            &mut module_rx,
11479            300,
11480            "aft",
11481            "released-before-mark",
11482        )
11483        .await;
11484        let (module_channel, module_epoch) = route_bind_channel(&bind);
11485        hold.wait().await;
11486
11487        handler.cleanup_connection(client).unwrap();
11488        assert!(
11489            module_rx.try_recv().is_err(),
11490            "client teardown must not GOODBYE a bind route.open is still queueing"
11491        );
11492
11493        hold.wait().await;
11494        let response = open_task.await.unwrap();
11495        assert_eq!(response.len(), 1);
11496        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11497
11498        let goodbye = module_rx
11499            .try_recv()
11500            .expect("module is told to drop the binding the daemon abandoned")
11501            .frame;
11502        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11503        assert_eq!(
11504            (goodbye.header.channel, goodbye.header.epoch),
11505            (module_channel, module_epoch)
11506        );
11507        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11508    }
11509
11510    /// The module acks inside the same window, but the client's egress closed
11511    /// first, so the reader refuses to commit. The ack proves the module holds
11512    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11513    /// send a second one.
11514    #[tokio::test]
11515    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11516        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11517        let client = ConnectionId::new(43);
11518        let (client_ctx, client_rx) = route_ctx(client);
11519
11520        let (open_task, bind) = relay_route_open(
11521            &handler,
11522            client,
11523            &client_ctx.egress,
11524            &mut module_rx,
11525            400,
11526            "aft",
11527            "gone-client-before-mark",
11528        )
11529        .await;
11530        let (module_channel, module_epoch) = route_bind_channel(&bind);
11531        hold.wait().await;
11532
11533        drop(client_rx);
11534        assert!(client_ctx.egress.is_closed());
11535        handler
11536            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11537            .await
11538            .unwrap();
11539
11540        hold.wait().await;
11541        let response = open_task.await.unwrap();
11542        assert_eq!(response.len(), 1);
11543        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11544
11545        let goodbye = module_rx
11546            .try_recv()
11547            .expect("module is told to drop the binding it acked")
11548            .frame;
11549        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11550        assert_eq!(
11551            (goodbye.header.channel, goodbye.header.epoch),
11552            (module_channel, module_epoch)
11553        );
11554        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11555        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11556    }
11557
11558    /// The fence at the module-loop boundary, stated as its own contract: which
11559    /// forwarding failures are allowed to end the module connection that is being
11560    /// served. A `ConnectionClosing` naming some client is about that client, and
11561    /// a module connection is shared; the same error naming the module's own
11562    /// connection is about this connection and must stay fatal, as must failures
11563    /// that are about the forwarding table itself.
11564    #[test]
11565    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11566        let handler = ControlHandler::default();
11567        let module_connection = ConnectionId::new(30);
11568        let client_connection = ConnectionId::new(31);
11569
11570        handler
11571            .refuse_to_end_module_connection_for_a_client(
11572                module_connection,
11573                77,
11574                ForwardingError::ConnectionClosing {
11575                    connection_id: client_connection,
11576                },
11577            )
11578            .expect("a closing client must never end the module connection");
11579
11580        assert!(matches!(
11581            handler.refuse_to_end_module_connection_for_a_client(
11582                module_connection,
11583                78,
11584                ForwardingError::ConnectionClosing {
11585                    connection_id: module_connection,
11586                },
11587            ),
11588            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11589                connection_id
11590            })) if connection_id == module_connection
11591        ));
11592        assert!(matches!(
11593            handler.refuse_to_end_module_connection_for_a_client(
11594                module_connection,
11595                79,
11596                ForwardingError::Poisoned,
11597            ),
11598            Err(RouterError::Forwarding(ForwardingError::Poisoned))
11599        ));
11600        assert!(matches!(
11601            handler.refuse_to_end_module_connection_for_a_client(
11602                module_connection,
11603                80,
11604                ForwardingError::StaleModuleEndpoint,
11605            ),
11606            Err(RouterError::Forwarding(
11607                ForwardingError::StaleModuleEndpoint
11608            ))
11609        ));
11610    }
11611
11612    /// The spawn-attestation guard is what stops a connected module from claiming
11613    /// another module's identity and being stamped `Reserved` for it. Every other
11614    /// test that supplies a consumer_identity supplies a CORRECT one, because a
11615    /// correct one is what the rest of the flow needs -- so the guard's rejection
11616    /// branch was never the subject of an assertion, only its acceptance branch.
11617    ///
11618    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
11619    /// whole subc-core library suite green; only the forwarding integration tests
11620    /// notice, and they notice for unrelated reasons. This test exists so the
11621    /// refusal itself is asserted where the guard lives: it fails if the identity
11622    /// check stops refusing, which is the direction that matters, since a guard
11623    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
11624    #[tokio::test]
11625    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11626        let registry = Arc::new(Registry::default());
11627        let forwarding = Arc::new(ForwardingTable::default());
11628        let supervisor = SupervisorHandle::new();
11629        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11630        let handler =
11631            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11632                .with_supervisor(supervisor);
11633
11634        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11635        hello_via_sink(
11636            &handler,
11637            &target_ctx,
11638            &mut target_rx,
11639            hello_frame("target", PROTOCOL_VERSION, 1),
11640        )
11641        .await;
11642
11643        // A real supervised module id presenting the wrong nonce. This is the
11644        // impersonation case: the attacker knows a privileged module_id, which is
11645        // public, and guesses at the nonce, which is not.
11646        let wrong_nonce = handler
11647            .handle_control_frame(
11648                &route_ctx(ConnectionId::new(91)).0,
11649                route_open_frame_with_admission_facts(
11650                    20,
11651                    "target",
11652                    unique_project_root("admission-facts"),
11653                    Some(subc_control::ConsumerIdentity {
11654                        module_id: "fed".to_string(),
11655                        launch_nonce: "not-the-real-nonce".to_string(),
11656                    }),
11657                    None,
11658                ),
11659            )
11660            .await
11661            .unwrap();
11662        assert_eq!(
11663            parse_error(&wrong_nonce[0])["code"],
11664            "bad_consumer_identity",
11665            "a mismatched launch nonce must be refused, not stamped Reserved"
11666        );
11667
11668        // A module id the supervisor never spawned at all, so no nonce exists to
11669        // compare against. An implementation that treats "no record" as "nothing
11670        // to check" fails open here while passing the case above.
11671        let never_spawned = handler
11672            .handle_control_frame(
11673                &route_ctx(ConnectionId::new(92)).0,
11674                route_open_frame_with_admission_facts(
11675                    21,
11676                    "target",
11677                    unique_project_root("admission-facts"),
11678                    Some(subc_control::ConsumerIdentity {
11679                        module_id: "never-spawned".to_string(),
11680                        launch_nonce: "any-nonce".to_string(),
11681                    }),
11682                    None,
11683                ),
11684            )
11685            .await
11686            .unwrap();
11687        assert_eq!(
11688            parse_error(&never_spawned[0])["code"],
11689            "bad_consumer_identity",
11690            "an unspawned module_id must be refused rather than accepted for lack of a record"
11691        );
11692    }
11693
11694    /// The refusal test above proves the guard says NO. Nothing proved it can say
11695    /// YES, and the difference is not academic: replacing the whole authorization
11696    /// with `false` -- admitting no consumer identity at all, revoking Reserved
11697    /// standing for every supervised module in the fleet -- leaves 110 of the 111
11698    /// library tests GREEN. The one that notices does so by HANGING, because it
11699    /// waits for a bind that can no longer happen.
11700    ///
11701    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
11702    /// or flaky test, invites a RETRY rather than an investigation, and the retry
11703    /// hangs too and gets blamed on the runner. So a total revocation of the
11704    /// daemon's trust grant would have shipped behind a symptom nobody attributes
11705    /// to code.
11706    ///
11707    /// The bias is structural rather than accidental. A REFUSAL looks like a
11708    /// failure someone writes a test for; a GRANT looks like the happy path. Every
11709    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
11710    /// suite for that reason, and this one is the purest case in the daemon.
11711    ///
11712    /// This test asserts the EFFECT rather than the absence of an error: the module
11713    /// receives a RouteBind and it carries `Reserved` naming the attested module.
11714    /// A guard that admitted nobody would produce no bind at all; one that admitted
11715    /// everybody would stamp the wrong principal, which the refusal test catches.
11716    #[tokio::test]
11717    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11718        let registry = Arc::new(Registry::default());
11719        let forwarding = Arc::new(ForwardingTable::default());
11720        let supervisor = SupervisorHandle::new();
11721        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11722        let handler =
11723            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11724                .with_supervisor(supervisor);
11725
11726        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11727        hello_via_sink(
11728            &handler,
11729            &target_ctx,
11730            &mut target_rx,
11731            hello_frame("target", PROTOCOL_VERSION, 1),
11732        )
11733        .await;
11734
11735        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11736        let route_handler = handler.clone();
11737        let route_task = tokio::spawn(async move {
11738            route_handler
11739                .handle_control_frame(
11740                    &client_ctx,
11741                    route_open_frame_with_admission_facts(
11742                        30,
11743                        "target",
11744                        unique_project_root("admission-facts"),
11745                        Some(subc_control::ConsumerIdentity {
11746                            module_id: "fed".to_string(),
11747                            launch_nonce: "fed-nonce".to_string(),
11748                        }),
11749                        None,
11750                    ),
11751                )
11752                .await
11753                .unwrap()
11754        });
11755
11756        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
11757        // the very mutation it exists to catch -- a guard that admits nobody -- no
11758        // bind is ever sent, so it HUNG rather than failing. That reproduces the
11759        // exact defect being fixed: a total revocation detected only as a stalled
11760        // suite, which reads as flakiness and invites a retry. An acceptance test
11761        // that waits for an effect must bound the wait, or a red becomes a hang.
11762        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11763            .await
11764            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11765            .expect("module control channel closed before route.bind");
11766        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11767        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11768            panic!("expected route.bind")
11769        };
11770        assert_eq!(
11771            principal,
11772            Some(Principal::Reserved {
11773                module_id: "fed".to_string()
11774            }),
11775            "a correctly attested consumer must be stamped Reserved for its own id"
11776        );
11777
11778        handler
11779            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11780            .await
11781            .unwrap();
11782        assert!(route_task.await.unwrap().is_empty());
11783        assert!(
11784            matches!(
11785                serde_json::from_slice::<ClientControlResponse>(
11786                    &client_rx.recv().await.unwrap().body
11787                )
11788                .unwrap(),
11789                ClientControlResponse::RouteOpen { .. }
11790            ),
11791            "the route must actually open, not merely avoid an error"
11792        );
11793    }
11794
11795    #[tokio::test(start_paused = true)]
11796    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11797        let registry = Arc::new(Registry::default());
11798        let forwarding = Arc::new(ForwardingTable::default());
11799        let supervisor = SupervisorHandle::new();
11800        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11801        let handler =
11802            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11803                .with_supervisor(supervisor);
11804
11805        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11806        hello_via_sink(
11807            &handler,
11808            &target_ctx,
11809            &mut target_rx,
11810            hello_frame("target", PROTOCOL_VERSION, 1),
11811        )
11812        .await;
11813
11814        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11815        let direct_handler = handler.clone();
11816        let direct_open = tokio::spawn(async move {
11817            direct_handler
11818                .handle_control_frame(
11819                    &direct_ctx,
11820                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
11821                )
11822                .await
11823                .unwrap()
11824        });
11825        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11826            .await
11827            .expect("no direct route.bind within 5s")
11828            .expect("target control channel closed before direct route.bind");
11829        handler
11830            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11831            .await
11832            .unwrap();
11833        assert!(direct_open.await.unwrap().is_empty());
11834        let _ = direct_rx.recv().await.unwrap();
11835
11836        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11837        let reserved_handler = handler.clone();
11838        let reserved_open = tokio::spawn(async move {
11839            reserved_handler
11840                .handle_control_frame(
11841                    &reserved_ctx,
11842                    route_open_frame_with_admission_facts(
11843                        3,
11844                        "target",
11845                        unique_project_root("admission-facts"),
11846                        Some(ConsumerIdentity {
11847                            module_id: "fed".to_string(),
11848                            launch_nonce: "fed-nonce".to_string(),
11849                        }),
11850                        None,
11851                    ),
11852                )
11853                .await
11854                .unwrap()
11855        });
11856        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11857            .await
11858            .expect("no reserved route.bind within 5s")
11859            .expect("target control channel closed before reserved route.bind");
11860        handler
11861            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11862            .await
11863            .unwrap();
11864        assert!(reserved_open.await.unwrap().is_empty());
11865        let _ = reserved_rx.recv().await.unwrap();
11866
11867        forwarding
11868            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11869            .unwrap();
11870        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11871        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11872            module_id: Some("target".to_string()),
11873        })
11874        .unwrap();
11875        let census_frame =
11876            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11877        let response = handler
11878            .handle_control_frame(&census_ctx, census_frame)
11879            .await
11880            .unwrap()
11881            .pop()
11882            .unwrap();
11883        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11884        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11885        assert!(matches!(
11886            decoded,
11887            ClientControlResponse::SupervisorRoutes { .. }
11888        ));
11889        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11890        assert_eq!(routes.len(), 2);
11891        assert!(routes.iter().all(|route| route["draining"] == true));
11892        // The census carries WHY: the reason the drain was begun with, in the
11893        // route.closing vocabulary, on every draining route this drain marked.
11894        assert!(
11895            routes.iter().all(|route| route["drain_reason"] == "reload"),
11896            "draining routes must name the drain's reason: {routes:?}"
11897        );
11898        assert!(routes.iter().any(|route| {
11899            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11900        }));
11901        assert!(routes.iter().any(|route| {
11902            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11903        }));
11904
11905        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11906            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11907        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11908            std::fs::write(
11909                &golden_path,
11910                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11911            )
11912            .unwrap();
11913        }
11914        let expected: Value =
11915            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11916        assert_eq!(actual, expected);
11917    }
11918
11919    async fn query_live_roots(
11920        handler: &ControlHandler,
11921        module_ctx: &RouteCtx,
11922    ) -> ModuleControlResponseToModule {
11923        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11924        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11925        let response = handler
11926            .handle_control_frame(module_ctx, frame)
11927            .await
11928            .unwrap()
11929            .pop()
11930            .unwrap();
11931        serde_json::from_slice(&response.body).unwrap()
11932    }
11933
11934    #[tokio::test(start_paused = true)]
11935    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11936        let registry = Arc::new(Registry::default());
11937        let forwarding = Arc::new(ForwardingTable::default());
11938        let handler = ControlHandler::with_forwarding(registry, forwarding);
11939        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11940        hello_via_sink(
11941            &handler,
11942            &target_ctx,
11943            &mut target_rx,
11944            hello_frame("target", PROTOCOL_VERSION, 1),
11945        )
11946        .await;
11947        let root = unique_project_root("live-roots-known");
11948        let path = ProjectRootId::from_path_allowing_missing(root.path())
11949            .unwrap()
11950            .as_path()
11951            .to_path_buf();
11952        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11953        let open_handler = handler.clone();
11954        let opened = tokio::spawn(async move {
11955            open_handler
11956                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11957                .await
11958                .unwrap()
11959        });
11960        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11961            .await
11962            .unwrap()
11963            .unwrap();
11964        handler
11965            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11966            .await
11967            .unwrap();
11968        assert!(opened.await.unwrap().is_empty());
11969        let _ = client_rx.recv().await.unwrap();
11970
11971        let root = unique_project_root("live-roots-pending");
11972        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11973            .unwrap()
11974            .as_path()
11975            .to_path_buf();
11976        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11977        let open_handler = handler.clone();
11978        let pending = tokio::spawn(async move {
11979            open_handler
11980                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11981                .await
11982                .unwrap()
11983        });
11984        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11985            .await
11986            .unwrap()
11987            .unwrap();
11988        let actual = query_live_roots(&handler, &target_ctx).await;
11989        let ModuleControlResponseToModule::LiveRoots {
11990            roots,
11991            unknown_root_bindings,
11992            total_bindings,
11993        } = actual
11994        else {
11995            panic!("expected live roots")
11996        };
11997        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11998        assert_eq!(unknown_root_bindings, 0);
11999        assert_eq!(
12000            roots.len(),
12001            2,
12002            "root-known arm must retain each canonical root"
12003        );
12004        assert_eq!(
12005            total_bindings,
12006            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12007        );
12008        let counts = roots
12009            .iter()
12010            .map(|root| (root.project_root.clone(), root.bound, root.pending))
12011            .collect::<Vec<_>>();
12012        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12013        expected.sort_by(|a, b| a.0.cmp(&b.0));
12014        assert_eq!(
12015            counts, expected,
12016            "roots must sort by path and count pending separately"
12017        );
12018        handler
12019            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12020            .await
12021            .unwrap();
12022        assert!(pending.await.unwrap().is_empty());
12023    }
12024
12025    #[tokio::test(start_paused = true)]
12026    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12027        let forwarding = Arc::new(ForwardingTable::default());
12028        let handler =
12029            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12030        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12031        hello_via_sink(
12032            &handler,
12033            &target_ctx,
12034            &mut target_rx,
12035            hello_frame("target", PROTOCOL_VERSION, 1),
12036        )
12037        .await;
12038        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12039        let pending = forwarding
12040            .begin_route_bind_relay_for_test(
12041                client_ctx.connection_id,
12042                client_ctx.egress.clone(),
12043                2,
12044                "target",
12045            )
12046            .unwrap();
12047        forwarding
12048            .complete_pending_relay(
12049                target_ctx.connection_id,
12050                pending.corr,
12051                RouteBindRelayOutcome::Accepted,
12052            )
12053            .unwrap();
12054        let actual = query_live_roots(&handler, &target_ctx).await;
12055        let ModuleControlResponseToModule::LiveRoots {
12056            roots,
12057            unknown_root_bindings,
12058            total_bindings,
12059        } = actual
12060        else {
12061            panic!("expected live roots")
12062        };
12063        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12064        assert_eq!(
12065            unknown_root_bindings, 1,
12066            "unknown-root arm must not read as no bindings"
12067        );
12068        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12069        assert_eq!(
12070            total_bindings,
12071            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12072        );
12073    }
12074
12075    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12076    /// so the ack has to be on its outbound queue before the module is
12077    /// routable. The connection loop writes a handler's replies only after the
12078    /// handler returns; this test stops in exactly that gap, runs a real
12079    /// route.open from another connection, and only then writes whatever the
12080    /// HELLO handler returned, the way the loop would. If the ack were still a
12081    /// reply, the route.bind request would reach the module first.
12082    #[tokio::test(start_paused = true)]
12083    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12084        let forwarding = Arc::new(ForwardingTable::default());
12085        let handler =
12086            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12087        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12088        let replies = handler
12089            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12090            .await
12091            .unwrap();
12092        let queued_by_hello = module_rx.len();
12093
12094        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12095        let open_handler = handler.clone();
12096        let open = tokio::spawn(async move {
12097            open_handler
12098                .handle_control_frame(
12099                    &client_ctx,
12100                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12101                )
12102                .await
12103                .unwrap()
12104        });
12105        // Let the route.open run until its route.bind is on the module's queue.
12106        let mut spins = 0;
12107        while module_rx.len() == queued_by_hello {
12108            spins += 1;
12109            assert!(spins < 10_000, "route.open never queued a route.bind");
12110            tokio::task::yield_now().await;
12111        }
12112
12113        // Now the connection loop's half: write the HELLO handler's replies.
12114        for reply in replies {
12115            module_ctx.egress.send(reply).await.unwrap();
12116        }
12117
12118        let first = module_rx.recv().await.unwrap().frame;
12119        assert_eq!(
12120            first.header.ty,
12121            FrameType::HelloAck,
12122            "the first frame a registering module reads must be its HELLO_ACK"
12123        );
12124        assert_eq!(first.header.corr, 7);
12125        let second = module_rx.recv().await.unwrap().frame;
12126        assert_eq!(second.header.ty, FrameType::Request);
12127        assert!(
12128            matches!(
12129                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12130                ModuleControlRequest::RouteBind { .. }
12131            ),
12132            "the route.bind follows the ack"
12133        );
12134        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12135
12136        handler
12137            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12138            .await
12139            .unwrap();
12140        assert!(open.await.unwrap().is_empty());
12141        let _ = client_rx.recv().await.unwrap();
12142    }
12143
12144    #[tokio::test(start_paused = true)]
12145    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12146        let handler = ControlHandler::with_forwarding(
12147            Arc::new(Registry::default()),
12148            Arc::new(ForwardingTable::default()),
12149        );
12150        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12151        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12152        hello_via_sink(
12153            &handler,
12154            &first_ctx,
12155            &mut first_rx,
12156            hello_frame("first", PROTOCOL_VERSION, 1),
12157        )
12158        .await;
12159        hello_via_sink(
12160            &handler,
12161            &second_ctx,
12162            &mut second_rx,
12163            hello_frame("second", PROTOCOL_VERSION, 2),
12164        )
12165        .await;
12166        let root = unique_project_root("second-only");
12167        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12168        let cloned = handler.clone();
12169        let open = tokio::spawn(async move {
12170            cloned
12171                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12172                .await
12173                .unwrap()
12174        });
12175        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12176            .await
12177            .unwrap()
12178            .unwrap();
12179        let first = query_live_roots(&handler, &first_ctx).await;
12180        let second = query_live_roots(&handler, &second_ctx).await;
12181        assert!(
12182            matches!(
12183                first,
12184                ModuleControlResponseToModule::LiveRoots {
12185                    total_bindings: 0,
12186                    ..
12187                }
12188            ),
12189            "cross-module scope must not expose another module's roots"
12190        );
12191        assert!(
12192            matches!(
12193                second,
12194                ModuleControlResponseToModule::LiveRoots {
12195                    total_bindings: 1,
12196                    ..
12197                }
12198            ),
12199            "second module must see its pending route"
12200        );
12201        handler
12202            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12203            .await
12204            .unwrap();
12205        assert!(open.await.unwrap().is_empty());
12206    }
12207
12208    #[tokio::test(start_paused = true)]
12209    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12210        let handler = ControlHandler::with_forwarding(
12211            Arc::new(Registry::default()),
12212            Arc::new(ForwardingTable::default()),
12213        );
12214        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12215        hello_via_sink(
12216            &handler,
12217            &target_ctx,
12218            &mut target_rx,
12219            hello_frame("target", PROTOCOL_VERSION, 1),
12220        )
12221        .await;
12222        let actual = query_live_roots(&handler, &target_ctx).await;
12223        let ModuleControlResponseToModule::LiveRoots {
12224            roots,
12225            unknown_root_bindings,
12226            total_bindings,
12227        } = actual
12228        else {
12229            panic!("expected live roots")
12230        };
12231        assert!(roots.is_empty());
12232        assert_eq!(unknown_root_bindings, 0);
12233        assert_eq!(total_bindings, 0);
12234        assert_eq!(
12235            total_bindings,
12236            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12237        );
12238    }
12239
12240    /// Read the vendored fed corpus rather than hand-building a package.
12241    ///
12242    /// A hand-built object encodes what the test author believed the carrier
12243    /// emits. These vectors are what it actually emits, and one of them exists
12244    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
12245    /// ignores additive unknown fields at the traversal emit terminus."
12246    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12247        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12248            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12249        let text = std::fs::read_to_string(&path)
12250            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12251        let vectors: Vec<(String, Value)> = text
12252            .lines()
12253            .filter(|line| !line.trim().is_empty())
12254            .map(|line| {
12255                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12256                let id = entry["corpus_id"]
12257                    .as_str()
12258                    .expect("every vector carries a corpus_id")
12259                    .to_string();
12260                (id, entry["package"].clone())
12261            })
12262            .collect();
12263        // Pin the count: a corpus that silently shrinks would take its coverage
12264        // with it, and a suite reading N-1 vectors reports the same clean pass
12265        // as one reading N.
12266        assert_eq!(
12267            vectors.len(),
12268            3,
12269            "vendored fed corpus changed size; re-sync from subc-federation"
12270        );
12271
12272        // Pin what makes the corpus DISCRIMINATING, not just present.
12273        //
12274        // The relay test below takes its expected value from the corpus, so the
12275        // corpus supplies the test's power to detect a lossy relay rather than
12276        // its correctness. A relay that dropped unrecognised fields would still
12277        // be caught -- but only by a package carrying fields it does not know.
12278        // Shrink every package to the handful of keys any implementation would
12279        // recognise and the test keeps passing over an input that can no longer
12280        // fail, which is the same clean green as a corpus that shrank away.
12281        //
12282        // So assert the precondition rather than duplicating the packages here:
12283        // at least one vector must carry a field beyond the small common set.
12284        // That is one claim to maintain instead of nine, and it fails loudly if
12285        // a re-sync ever flattens the corpus.
12286        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12287        let richest = vectors
12288            .iter()
12289            .filter_map(|(_, package)| package.as_object())
12290            .map(|object| {
12291                object
12292                    .keys()
12293                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12294                    .count()
12295            })
12296            .max()
12297            .unwrap_or(0);
12298        assert!(
12299            richest >= 2,
12300            "vendored corpus no longer carries a package with unmodelled fields, \
12301             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12302        );
12303
12304        vectors
12305    }
12306
12307    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12308    ///
12309    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12310    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12311    /// three-key object, so a relay that quietly dropped fields it did not
12312    /// recognise would satisfy it. These vectors carry nine keys including ones
12313    /// this crate has no type for, so a typed relay fails here and only here.
12314    #[tokio::test]
12315    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12316        for (corpus_id, package) in fed_admission_facts_vectors() {
12317            let registry = Arc::new(Registry::default());
12318            let forwarding = Arc::new(ForwardingTable::default());
12319            let supervisor = SupervisorHandle::new();
12320            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12321            let handler =
12322                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12323                    .with_supervisor(supervisor)
12324                    .with_admission_facts_config(
12325                        Some("fed".to_string()),
12326                        Some(vec!["target".to_string()]),
12327                    );
12328
12329            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12330            hello_via_sink(
12331                &handler,
12332                &target_ctx,
12333                &mut target_rx,
12334                hello_frame("target", PROTOCOL_VERSION, 1),
12335            )
12336            .await;
12337
12338            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12339            let route_handler = handler.clone();
12340            let expected = package.clone();
12341            let route_task = tokio::spawn(async move {
12342                route_handler
12343                    .handle_control_frame(
12344                        &client_ctx,
12345                        route_open_frame_with_admission_facts(
12346                            20,
12347                            "target",
12348                            unique_project_root("admission-facts"),
12349                            Some(subc_control::ConsumerIdentity {
12350                                module_id: "fed".to_string(),
12351                                launch_nonce: "fed-nonce".to_string(),
12352                            }),
12353                            Some(package),
12354                        ),
12355                    )
12356                    .await
12357                    .unwrap()
12358            });
12359
12360            let bind_frame = target_rx.recv().await.unwrap();
12361            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12362            let ModuleControlRequest::RouteBind {
12363                admission_facts, ..
12364            } = bind
12365            else {
12366                panic!("{corpus_id}: expected route.bind")
12367            };
12368            assert_eq!(
12369                admission_facts,
12370                Some(expected),
12371                "{corpus_id}: relay must not add, drop or reshape any field"
12372            );
12373
12374            handler
12375                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12376                .await
12377                .unwrap();
12378            route_task.await.unwrap();
12379        }
12380    }
12381
12382    #[tokio::test]
12383    async fn route_bind_never_relays_the_openers_build_class() {
12384        let registry = Arc::new(Registry::default());
12385        let forwarding = Arc::new(ForwardingTable::default());
12386        let handler =
12387            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12388        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12389        hello_via_sink(
12390            &handler,
12391            &target_ctx,
12392            &mut target_rx,
12393            hello_frame("target", PROTOCOL_VERSION, 1),
12394        )
12395        .await;
12396
12397        let root = unique_project_root("build-class");
12398        let mut identity = BindIdentity::new(
12399            root.path().to_path_buf(),
12400            "unit".to_string(),
12401            "s".to_string(),
12402        );
12403        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12404        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12405            target: RouteTarget::ToolProvider {
12406                module_id: "target".to_string(),
12407            },
12408            identity,
12409            consumer_identity: None,
12410            consumer_capabilities: None,
12411            role_versions: None,
12412            admission_facts: None,
12413            scope: None,
12414        })
12415        .unwrap();
12416        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12417
12418        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12419        let route_handler = handler.clone();
12420        let route_task = tokio::spawn(async move {
12421            route_handler
12422                .handle_control_frame(&client_ctx, open)
12423                .await
12424                .unwrap()
12425        });
12426        let bind_frame = target_rx.recv().await.unwrap();
12427        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12428        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12429            panic!("expected route.bind")
12430        };
12431        assert_eq!(
12432            identity.build_class, None,
12433            "route.bind carried the build class"
12434        );
12435        assert!(
12436            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12437            "the route.bind bytes still name build_class"
12438        );
12439        handler
12440            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12441            .await
12442            .unwrap();
12443        route_task.await.unwrap();
12444    }
12445
12446    #[tokio::test]
12447    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12448        let registry = Arc::new(Registry::default());
12449        let forwarding = Arc::new(ForwardingTable::default());
12450        let supervisor = SupervisorHandle::new();
12451        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12452        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12453        let handler =
12454            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12455                .with_supervisor(supervisor)
12456                .with_admission_facts_config(
12457                    Some("fed".to_string()),
12458                    Some(vec!["target".to_string()]),
12459                );
12460
12461        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12462        hello_via_sink(
12463            &handler,
12464            &target_ctx,
12465            &mut target_rx,
12466            hello_frame("target", PROTOCOL_VERSION, 1),
12467        )
12468        .await;
12469        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12470        hello_via_sink(
12471            &handler,
12472            &other_ctx,
12473            &mut other_rx,
12474            hello_frame("other", PROTOCOL_VERSION, 2),
12475        )
12476        .await;
12477
12478        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12479        let expected_facts = facts.clone();
12480        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12481        let route_handler = handler.clone();
12482        let route_task = tokio::spawn(async move {
12483            route_handler
12484                .handle_control_frame(
12485                    &client_ctx,
12486                    route_open_frame_with_admission_facts(
12487                        10,
12488                        "target",
12489                        unique_project_root("admission-facts"),
12490                        Some(subc_control::ConsumerIdentity {
12491                            module_id: "fed".to_string(),
12492                            launch_nonce: "fed-nonce".to_string(),
12493                        }),
12494                        Some(facts.clone()),
12495                    ),
12496                )
12497                .await
12498                .unwrap()
12499        });
12500        let bind_frame = target_rx.recv().await.unwrap();
12501        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12502        let ModuleControlRequest::RouteBind {
12503            admission_facts, ..
12504        } = bind
12505        else {
12506            panic!("expected route.bind")
12507        };
12508        assert_eq!(admission_facts, Some(expected_facts));
12509        handler
12510            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12511            .await
12512            .unwrap();
12513        assert!(route_task.await.unwrap().is_empty());
12514        assert!(matches!(
12515            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12516                .unwrap(),
12517            ClientControlResponse::RouteOpen { .. }
12518        ));
12519
12520        let direct = handler
12521            .handle_control_frame(
12522                &route_ctx(ConnectionId::new(73)).0,
12523                route_open_frame_with_admission_facts(
12524                    11,
12525                    "target",
12526                    unique_project_root("admission-facts"),
12527                    None,
12528                    Some(json!({"x": 1})),
12529                ),
12530            )
12531            .await
12532            .unwrap();
12533        assert_eq!(
12534            parse_error(&direct[0])["code"],
12535            "admission_facts_not_permitted"
12536        );
12537
12538        let different_reserved = handler
12539            .handle_control_frame(
12540                &route_ctx(ConnectionId::new(77)).0,
12541                route_open_frame_with_admission_facts(
12542                    15,
12543                    "target",
12544                    unique_project_root("admission-facts"),
12545                    Some(subc_control::ConsumerIdentity {
12546                        module_id: "other".to_string(),
12547                        launch_nonce: "other-nonce".to_string(),
12548                    }),
12549                    Some(json!({"x": 1})),
12550                ),
12551            )
12552            .await
12553            .unwrap();
12554        assert_eq!(
12555            parse_error(&different_reserved[0])["code"],
12556            "admission_facts_not_permitted"
12557        );
12558
12559        let other_target = handler
12560            .handle_control_frame(
12561                &route_ctx(ConnectionId::new(74)).0,
12562                route_open_frame_with_admission_facts(
12563                    12,
12564                    "other",
12565                    unique_project_root("admission-facts"),
12566                    Some(subc_control::ConsumerIdentity {
12567                        module_id: "fed".to_string(),
12568                        launch_nonce: "fed-nonce".to_string(),
12569                    }),
12570                    Some(json!({"x": 1})),
12571                ),
12572            )
12573            .await
12574            .unwrap();
12575        assert_eq!(
12576            parse_error(&other_target[0])["code"],
12577            "admission_facts_target_not_allowed"
12578        );
12579
12580        let nonexistent = handler
12581            .handle_control_frame(
12582                &route_ctx(ConnectionId::new(75)).0,
12583                route_open_frame_with_admission_facts(
12584                    13,
12585                    "missing",
12586                    unique_project_root("admission-facts"),
12587                    None,
12588                    Some(json!({"x": 1})),
12589                ),
12590            )
12591            .await
12592            .unwrap();
12593        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12594
12595        let described = handler
12596            .handle_control_frame(
12597                &route_ctx(ConnectionId::new(76)).0,
12598                Frame::build(
12599                    FrameType::Request,
12600                    control_flags(),
12601                    0,
12602                    0,
12603                    14,
12604                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12605                )
12606                .unwrap(),
12607            )
12608            .await
12609            .unwrap();
12610        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12611            serde_json::from_slice(&described[0].body).unwrap()
12612        else {
12613            panic!("expected server.describe response")
12614        };
12615        assert!(capabilities
12616            .iter()
12617            .any(|cap| cap == "admission_facts_relay_v1"));
12618    }
12619
12620    #[tokio::test]
12621    async fn admission_facts_without_configured_carrier_are_rejected() {
12622        let registry = Arc::new(Registry::default());
12623        let forwarding = Arc::new(ForwardingTable::default());
12624        let handler = ControlHandler::with_forwarding(registry, forwarding);
12625        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12626        hello_via_sink(
12627            &handler,
12628            &target_ctx,
12629            &mut target_rx,
12630            hello_frame("target", PROTOCOL_VERSION, 1),
12631        )
12632        .await;
12633
12634        let responses = handler
12635            .handle_control_frame(
12636                &route_ctx(ConnectionId::new(79)).0,
12637                route_open_frame_with_admission_facts(
12638                    16,
12639                    "target",
12640                    unique_project_root("admission-facts"),
12641                    None,
12642                    Some(json!({"x": 1})),
12643                ),
12644            )
12645            .await
12646            .unwrap();
12647        assert_eq!(
12648            parse_error(&responses[0])["code"],
12649            "admission_facts_not_permitted"
12650        );
12651    }
12652
12653    #[tokio::test]
12654    async fn route_open_relays_consumer_capabilities_verbatim() {
12655        let registry = Arc::new(Registry::default());
12656        let forwarding = Arc::new(ForwardingTable::default());
12657        let handler =
12658            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12659        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12660        hello_via_sink(
12661            &handler,
12662            &module_ctx,
12663            &mut module_rx,
12664            hello_frame("aft", PROTOCOL_VERSION, 7),
12665        )
12666        .await;
12667
12668        let expected = vec!["elicitation".to_string(), "roots".to_string()];
12669        let expected_for_request = expected.clone();
12670        let project_root = unique_project_root("consumer-capabilities-present");
12671        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12672        let route_handler = handler.clone();
12673        let route_task = tokio::spawn(async move {
12674            route_handler
12675                .handle_control_frame(
12676                    &client_ctx,
12677                    route_open_frame_with_consumer_capabilities(
12678                        401,
12679                        "aft",
12680                        project_root,
12681                        Some(expected_for_request),
12682                    ),
12683                )
12684                .await
12685                .unwrap()
12686        });
12687        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12688            .await
12689            .unwrap()
12690            .unwrap();
12691        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12692        let ModuleControlRequest::RouteBind {
12693            consumer_capabilities,
12694            ..
12695        } = bind
12696        else {
12697            panic!("expected route.bind request, got {bind:?}");
12698        };
12699        assert_eq!(consumer_capabilities, Some(expected.clone()));
12700
12701        handler
12702            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12703            .await
12704            .unwrap();
12705        let route_response = route_task.await.unwrap();
12706        assert!(route_response.is_empty());
12707        let published = client_rx.recv().await.unwrap();
12708        assert!(matches!(
12709            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12710            ClientControlResponse::RouteOpen { .. }
12711        ));
12712    }
12713
12714    #[tokio::test]
12715    async fn route_open_without_consumer_capabilities_relays_none() {
12716        let registry = Arc::new(Registry::default());
12717        let forwarding = Arc::new(ForwardingTable::default());
12718        let handler =
12719            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12720        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12721        hello_via_sink(
12722            &handler,
12723            &module_ctx,
12724            &mut module_rx,
12725            hello_frame("aft", PROTOCOL_VERSION, 7),
12726        )
12727        .await;
12728
12729        let project_root = unique_project_root("consumer-capabilities-absent");
12730        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12731        let route_handler = handler.clone();
12732        let route_task = tokio::spawn(async move {
12733            route_handler
12734                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12735                .await
12736                .unwrap()
12737        });
12738        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12739            .await
12740            .unwrap()
12741            .unwrap();
12742        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12743        let ModuleControlRequest::RouteBind {
12744            consumer_capabilities,
12745            ..
12746        } = bind
12747        else {
12748            panic!("expected route.bind request, got {bind:?}");
12749        };
12750        assert_eq!(consumer_capabilities, None);
12751
12752        handler
12753            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12754            .await
12755            .unwrap();
12756        let route_response = route_task.await.unwrap();
12757        assert!(route_response.is_empty());
12758        let published = client_rx.recv().await.unwrap();
12759        assert!(matches!(
12760            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12761            ClientControlResponse::RouteOpen { .. }
12762        ));
12763    }
12764
12765    /// Opens a route to a freshly registered `aft` with `sent` as the
12766    /// route.open's role_versions, acks the bind, and returns the role_versions
12767    /// the module's bind carried.
12768    async fn bind_role_versions_for(
12769        sent: Option<BTreeMap<String, String>>,
12770        connection: u64,
12771    ) -> Option<BTreeMap<String, String>> {
12772        let registry = Arc::new(Registry::default());
12773        let forwarding = Arc::new(ForwardingTable::default());
12774        let handler =
12775            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12776        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12777        hello_via_sink(
12778            &handler,
12779            &module_ctx,
12780            &mut module_rx,
12781            hello_frame("aft", PROTOCOL_VERSION, 7),
12782        )
12783        .await;
12784        let project_root = unique_project_root("role-versions");
12785        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12786        let route_handler = handler.clone();
12787        let route_task = tokio::spawn(async move {
12788            route_handler
12789                .handle_control_frame(
12790                    &client_ctx,
12791                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
12792                )
12793                .await
12794                .unwrap()
12795        });
12796        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12797            .await
12798            .expect("a well-formed route.open reaches the module as a bind")
12799            .unwrap();
12800        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12801        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12802            panic!("expected route.bind request, got {bind:?}");
12803        };
12804        handler
12805            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12806            .await
12807            .unwrap();
12808        assert!(route_task.await.unwrap().is_empty());
12809        let published = client_rx.recv().await.unwrap();
12810        assert!(matches!(
12811            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12812            ClientControlResponse::RouteOpen { .. }
12813        ));
12814        role_versions
12815    }
12816
12817    #[tokio::test]
12818    async fn route_open_relays_role_versions_verbatim() {
12819        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12820        assert_eq!(
12821            bind_role_versions_for(Some(sent.clone()), 141).await,
12822            Some(sent)
12823        );
12824    }
12825
12826    /// An empty map declares nothing, so the provider sees no field rather
12827    /// than an empty object it would have to treat as a second "none".
12828    #[tokio::test]
12829    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12830        assert_eq!(bind_role_versions_for(None, 143).await, None);
12831        assert_eq!(
12832            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12833            None
12834        );
12835    }
12836
12837    #[tokio::test]
12838    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12839        let registry = Arc::new(Registry::default());
12840        let forwarding = Arc::new(ForwardingTable::default());
12841        let handler =
12842            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12843        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12844        hello_via_sink(
12845            &handler,
12846            &module_ctx,
12847            &mut module_rx,
12848            hello_frame("aft", PROTOCOL_VERSION, 7),
12849        )
12850        .await;
12851
12852        let nine: BTreeMap<String, String> = (0..9)
12853            .map(|index| (format!("role-{index}"), "v1".to_string()))
12854            .collect();
12855        for (label, malformed) in [
12856            (
12857                "invalid role name",
12858                role_versions(&[("Tool_Provider", "v1")]),
12859            ),
12860            ("invalid version", role_versions(&[("tool-provider", "v0")])),
12861            ("nine entries", nine),
12862        ] {
12863            // A refused open answers at once; one that reached the module
12864            // would wait for its bind ack and trip this timeout.
12865            let responses = tokio::time::timeout(
12866                Duration::from_secs(1),
12867                handler.handle_control_frame(
12868                    &route_ctx(ConnectionId::new(148)).0,
12869                    route_open_frame_with_role_versions(
12870                        404,
12871                        "aft",
12872                        unique_project_root("role-versions-malformed"),
12873                        Some(malformed),
12874                    ),
12875                ),
12876            )
12877            .await
12878            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12879            .unwrap();
12880            assert_eq!(responses.len(), 1, "{label}");
12881            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12882            let error = parse_error(&responses[0]);
12883            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12884            assert_eq!(
12885                error["detail"]["field"], "role_versions",
12886                "{label}: {error}"
12887            );
12888            assert!(
12889                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12890                "{label}: a malformed declaration is terminal"
12891            );
12892            assert!(
12893                module_rx.try_recv().is_err(),
12894                "{label}: the module must never see a bind"
12895            );
12896        }
12897    }
12898
12899    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
12900    /// consumer can tell this daemon forwards the field from one that would
12901    /// drop it.
12902    #[tokio::test]
12903    async fn route_role_versions_capability_is_advertised() {
12904        let handler = ControlHandler::new(Arc::new(Registry::default()));
12905        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12906        let ack = hello_via_sink(
12907            &handler,
12908            &ctx,
12909            &mut rx,
12910            hello_frame("m", PROTOCOL_VERSION, 1),
12911        )
12912        .await;
12913        let ack = parse_ack(&ack);
12914        assert!(
12915            ack.subc_capabilities
12916                .iter()
12917                .any(|c| c == "route-role-versions/v1"),
12918            "{:?}",
12919            ack.subc_capabilities
12920        );
12921
12922        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12923        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12924        let reply = handler
12925            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12926            .await
12927            .unwrap()
12928            .pop()
12929            .unwrap();
12930        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12931            serde_json::from_slice(&reply.body).unwrap()
12932        else {
12933            panic!("not a server.describe reply");
12934        };
12935        assert!(
12936            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12937            "{capabilities:?}"
12938        );
12939    }
12940
12941    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12942    /// catalog entry without `events` (the module declares none) from a daemon
12943    /// that never reports declared events.
12944    #[tokio::test]
12945    async fn catalog_events_capability_is_advertised() {
12946        let handler = ControlHandler::new(Arc::new(Registry::default()));
12947        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12948        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12949        let reply = handler
12950            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12951            .await
12952            .unwrap()
12953            .pop()
12954            .unwrap();
12955        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12956            serde_json::from_slice(&reply.body).unwrap()
12957        else {
12958            panic!("not a server.describe reply");
12959        };
12960        assert!(
12961            capabilities
12962                .iter()
12963                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12964            "{capabilities:?}"
12965        );
12966    }
12967
12968    #[tokio::test]
12969    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12970        let handler = ControlHandler::new(Arc::new(Registry::default()));
12971        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12972        let ack = hello_via_sink(
12973            &handler,
12974            &module,
12975            &mut module_rx,
12976            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12977        )
12978        .await;
12979        let ack = parse_ack(&ack);
12980        assert!(
12981            ack.subc_capabilities
12982                .iter()
12983                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12984            "{:?}",
12985            ack.subc_capabilities
12986        );
12987        assert!(ack
12988            .subc_capabilities
12989            .iter()
12990            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12991
12992        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12993        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12994        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12995        let reply = handler
12996            .handle_control_frame(&client, frame)
12997            .await
12998            .unwrap()
12999            .pop()
13000            .unwrap();
13001        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13002            serde_json::from_slice(&reply.body).unwrap()
13003        else {
13004            panic!("not a server.describe reply");
13005        };
13006        assert!(
13007            capabilities
13008                .iter()
13009                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13010            "{capabilities:?}"
13011        );
13012        assert!(capabilities
13013            .iter()
13014            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13015    }
13016
13017    #[tokio::test]
13018    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13019        let handler = ControlHandler::new(Arc::new(Registry::default()));
13020        let (module, mut rx) = route_ctx(ConnectionId::new(156));
13021        let ack = hello_via_sink(
13022            &handler,
13023            &module,
13024            &mut rx,
13025            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13026        )
13027        .await;
13028        assert!(parse_ack(&ack)
13029            .subc_capabilities
13030            .iter()
13031            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13032        assert!(parse_ack(&ack)
13033            .subc_capabilities
13034            .iter()
13035            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13036        let (client, _rx) = route_ctx(ConnectionId::new(157));
13037        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13038        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13039        let reply = handler
13040            .handle_control_frame(&client, frame)
13041            .await
13042            .unwrap()
13043            .pop()
13044            .unwrap();
13045        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13046            serde_json::from_slice(&reply.body).unwrap()
13047        else {
13048            panic!("server.describe reply")
13049        };
13050        assert!(capabilities
13051            .iter()
13052            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13053        assert!(capabilities
13054            .iter()
13055            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13056    }
13057
13058    #[tokio::test]
13059    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13060        let handler = ControlHandler::new(Arc::new(Registry::default()));
13061        let (module, mut rx) = route_ctx(ConnectionId::new(158));
13062        let ack = hello_via_sink(
13063            &handler,
13064            &module,
13065            &mut rx,
13066            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13067        )
13068        .await;
13069        assert!(parse_ack(&ack)
13070            .subc_capabilities
13071            .iter()
13072            .any(|cap| cap == "scope-workdir/v1"));
13073        let (client, _rx) = route_ctx(ConnectionId::new(159));
13074        let frame = Frame::build(
13075            FrameType::Request,
13076            control_flags(),
13077            0,
13078            0,
13079            5,
13080            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13081        )
13082        .unwrap();
13083        let reply = handler
13084            .handle_control_frame(&client, frame)
13085            .await
13086            .unwrap()
13087            .remove(0);
13088        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13089            serde_json::from_slice(&reply.body).unwrap()
13090        else {
13091            panic!("server.describe reply")
13092        };
13093        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13094    }
13095
13096    #[tokio::test]
13097    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13098        let handler = ControlHandler::new(Arc::new(Registry::default()));
13099        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13100        let ack = hello_via_sink(
13101            &handler,
13102            &module,
13103            &mut rx,
13104            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13105        )
13106        .await;
13107        assert!(parse_ack(&ack)
13108            .subc_capabilities
13109            .iter()
13110            .any(|cap| cap == "worker-runs/v1"));
13111        let (client, _rx) = route_ctx(ConnectionId::new(161));
13112        let frame = Frame::build(
13113            FrameType::Request,
13114            control_flags(),
13115            0,
13116            0,
13117            5,
13118            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13119        )
13120        .unwrap();
13121        let reply = handler
13122            .handle_control_frame(&client, frame)
13123            .await
13124            .unwrap()
13125            .remove(0);
13126        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13127            serde_json::from_slice(&reply.body).unwrap()
13128        else {
13129            panic!("server.describe reply")
13130        };
13131        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13132    }
13133
13134    #[tokio::test]
13135    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13136        let handler = ControlHandler::new(Arc::new(Registry::default()));
13137        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13138        let ack = hello_via_sink(
13139            &handler,
13140            &module,
13141            &mut rx,
13142            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13143        )
13144        .await;
13145        assert!(parse_ack(&ack)
13146            .subc_capabilities
13147            .iter()
13148            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13149        let (client, _rx) = route_ctx(ConnectionId::new(155));
13150        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13151        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13152        let reply = handler
13153            .handle_control_frame(&client, frame)
13154            .await
13155            .unwrap()
13156            .pop()
13157            .unwrap();
13158        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13159            serde_json::from_slice(&reply.body).unwrap()
13160        else {
13161            panic!("not a server.describe reply")
13162        };
13163        assert!(capabilities
13164            .iter()
13165            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13166    }
13167
13168    #[tokio::test]
13169    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13170        let registry = Arc::new(Registry::default());
13171        let forwarding = Arc::new(ForwardingTable::default());
13172        let handler =
13173            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13174                .with_health_probe_timeout(Duration::from_secs(5));
13175        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13176        hello_via_sink(
13177            &handler,
13178            &module_ctx,
13179            &mut module_rx,
13180            non_routable_hello_frame_with_control_ops(
13181                "mcp",
13182                300,
13183                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13184            ),
13185        )
13186        .await;
13187        assert!(registry
13188            .get_module("mcp")
13189            .unwrap()
13190            .unwrap()
13191            .manifest
13192            .provides
13193            .is_empty());
13194
13195        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13196        let route_response = handler
13197            .handle_control_frame(
13198                &route_client_ctx,
13199                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13200            )
13201            .await
13202            .unwrap();
13203        assert_eq!(route_response[0].header.ty, FrameType::Error);
13204        assert_eq!(
13205            parse_error(&route_response[0])["code"],
13206            "target_unavailable"
13207        );
13208        assert!(parse_error(&route_response[0])["message"]
13209            .as_str()
13210            .unwrap()
13211            .contains("does not provide the requested target"));
13212        assert!(module_rx.try_recv().is_err());
13213
13214        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13215        let health_handler = handler.clone();
13216        let health_task = tokio::spawn(async move {
13217            health_handler
13218                .handle_control_frame(
13219                    &health_client_ctx,
13220                    supervisor_health_probe_frame(302, "mcp"),
13221                )
13222                .await
13223                .unwrap()
13224        });
13225        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13226            .await
13227            .unwrap()
13228            .unwrap();
13229        assert_eq!(
13230            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13231            ModuleControlRequest::HealthCheck {}
13232        );
13233        handler
13234            .handle_control_frame(
13235                &module_ctx,
13236                health_response(health_frame.header.corr, HealthStatus::Ok),
13237            )
13238            .await
13239            .unwrap();
13240        let health_response = health_task.await.unwrap();
13241        assert_eq!(health_response[0].header.ty, FrameType::Response);
13242        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13243            ClientControlResponse::SupervisorHealthProbe {
13244                module_id, status, ..
13245            } => {
13246                assert_eq!(module_id, "mcp");
13247                assert_eq!(status, HealthStatus::Ok);
13248            }
13249            other => panic!("unexpected health response: {other:?}"),
13250        }
13251
13252        // Exercise the forwarding cleanup path directly while leaving the registry
13253        // advertisement in place. If cleanup leaves a stale control sink behind,
13254        // the next probe will enqueue onto it and wait for the long probe timeout
13255        // instead of returning an immediate no-connection error.
13256        forwarding
13257            .cleanup_connection(module_ctx.connection_id)
13258            .unwrap();
13259        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13260        let cleanup_response = tokio::time::timeout(
13261            Duration::from_millis(200),
13262            handler.handle_control_frame(
13263                &cleanup_probe_ctx,
13264                supervisor_health_probe_frame(303, "mcp"),
13265            ),
13266        )
13267        .await
13268        .expect("probe should fail immediately when the control lane is gone")
13269        .unwrap();
13270        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13271        assert_eq!(
13272            parse_error(&cleanup_response[0])["code"],
13273            "target_unavailable"
13274        );
13275        assert!(parse_error(&cleanup_response[0])["message"]
13276            .as_str()
13277            .unwrap()
13278            .contains("no module connection"));
13279
13280        handler
13281            .cleanup_connection(module_ctx.connection_id)
13282            .unwrap();
13283    }
13284
13285    #[tokio::test]
13286    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13287        let registry = Arc::new(Registry::default());
13288        let supervisor_handle = SupervisorHandle::new();
13289        let supervisor =
13290            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13291                .with_handle(supervisor_handle.clone())
13292                .with_connection_file_path(
13293                    std::env::temp_dir()
13294                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13295                );
13296        let module = supervisor
13297            .supervise_configured(
13298                ModuleSpec {
13299                    module_id: "warming".to_string(),
13300                    program: fake_aft_stub_path(),
13301                    args: Vec::new(),
13302                    env: Vec::new(),
13303                    reserved: false,
13304                    reserved_prefixes: Vec::new(),
13305                    protocol: ModuleProtocol::Subc,
13306                    overlap: Default::default(),
13307                },
13308                true,
13309            )
13310            .unwrap();
13311        assert_eq!(module.state().unwrap(), ModuleState::Running);
13312
13313        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13314        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13315        let response = handler
13316            .handle_control_frame(
13317                &ctx,
13318                route_open_frame(304, "warming", unique_project_root("warming")),
13319            )
13320            .await
13321            .unwrap();
13322        module.stop().await.unwrap();
13323
13324        assert_eq!(response[0].header.ty, FrameType::Error);
13325        let error = parse_error(&response[0]);
13326        assert_eq!(error["code"], "module_warming");
13327        assert!(error["message"]
13328            .as_str()
13329            .unwrap()
13330            .contains("state=running, enabled=true, live=false"));
13331    }
13332
13333    #[test]
13334    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13335        let handler = ControlHandler::new(Arc::new(Registry::default()));
13336        let capture = EventCapture::default();
13337        let _subscriber =
13338            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13339        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13340        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13341        let pending = (0..limit).collect::<Vec<_>>();
13342        let response = handler
13343            .route_open_capacity_refusal(
13344                &ctx,
13345                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13346                "busy",
13347                pending.len(),
13348                limit,
13349            )
13350            .unwrap();
13351        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13352        let event = capture
13353            .events()
13354            .into_iter()
13355            .find(|event| {
13356                event.target == "control"
13357                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13358            })
13359            .expect("connection admission refusal event");
13360        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13361        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13362        // Refused before `handle_route_open` parses the request, so the caller
13363        // fields are read from the frame itself.
13364        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13365        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13366    }
13367
13368    #[test]
13369    fn route_open_target_cap_logs_admission_reason_and_capacity() {
13370        let handler = ControlHandler::new(Arc::new(Registry::default()));
13371        let capture = EventCapture::default();
13372        let _subscriber =
13373            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13374        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13375        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13376        let guards = (0..limit)
13377            .map(|_| {
13378                handler
13379                    .route_bind_concurrency
13380                    .try_admit("busy", limit)
13381                    .unwrap()
13382            })
13383            .collect::<Vec<_>>();
13384        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13385            Err(in_flight) => in_flight,
13386            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13387        };
13388        let response = handler
13389            .route_open_target_capacity_refusal(
13390                &ctx,
13391                &RouteOpenCaller::unknown(),
13392                &route_open_frame(397, "busy", unique_project_root("target-cap")),
13393                "busy",
13394                in_flight,
13395            )
13396            .unwrap();
13397        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13398        let event = capture
13399            .events()
13400            .into_iter()
13401            .find(|event| {
13402                event.target == "control"
13403                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13404            })
13405            .expect("target admission refusal event");
13406        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13407        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13408        drop(guards);
13409    }
13410
13411    #[cfg(unix)]
13412    #[tokio::test(start_paused = true)]
13413    async fn supervisor_list_shows_a_failed_modules_next_retry() {
13414        let registry = Arc::new(Registry::default());
13415        let handle = SupervisorHandle::new();
13416        let supervisor =
13417            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13418                .with_handle(handle.clone());
13419        let module = supervisor
13420            .spawn(crate::ModuleSpec {
13421                module_id: "retry-list".into(),
13422                program: PathBuf::from("/bin/sh"),
13423                args: vec!["-c".into(), "exit 17".into()],
13424                env: vec![],
13425                reserved: false,
13426                reserved_prefixes: vec![],
13427                protocol: ModuleProtocol::None,
13428                overlap: Default::default(),
13429            })
13430            .unwrap();
13431        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13432        while module.state().unwrap() != crate::ModuleState::Failed {
13433            assert!(
13434                std::time::Instant::now() < deadline,
13435                "waiting for budget failure"
13436            );
13437            tokio::task::yield_now().await;
13438        }
13439        let handler = ControlHandler::new(registry).with_supervisor(handle);
13440        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13441        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13442        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13443        assert_eq!(body["modules"][0]["state"], "failed");
13444        let retry = body["modules"][0]["next_retry_at_ms"]
13445            .as_u64()
13446            .expect("scheduled retry on the wire");
13447        let now = std::time::SystemTime::now()
13448            .duration_since(std::time::UNIX_EPOCH)
13449            .unwrap()
13450            .as_millis() as u64;
13451        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13452        module.set_enabled(false).await.unwrap();
13453        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13454        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13455        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13456        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13457    }
13458
13459    #[tokio::test]
13460    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13461        let registry = Arc::new(Registry::default());
13462        let handle = SupervisorHandle::new();
13463        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13464            .with_handle(handle.clone());
13465        supervisor
13466            .supervise_configured(
13467                crate::ModuleSpec {
13468                    module_id: "drift".into(),
13469                    program: PathBuf::from("disabled-module"),
13470                    args: Vec::new(),
13471                    env: Vec::new(),
13472                    reserved: false,
13473                    reserved_prefixes: Vec::new(),
13474                    protocol: ModuleProtocol::Subc,
13475                    overlap: Default::default(),
13476                },
13477                false,
13478            )
13479            .unwrap();
13480        let handler = ControlHandler::new(registry).with_supervisor(handle);
13481        for configured in [None, Some(false), Some(true)] {
13482            handler.capability_evaluator.configure(
13483                configured.map(|enabled| ("drift".to_string(), enabled)),
13484                BTreeMap::new(),
13485            );
13486            let frame =
13487                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13488            let replies = handler
13489                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13490                .await
13491                .unwrap();
13492            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13493            assert!(body.get("configured_enabled").is_none());
13494            assert_eq!(body["op"], "supervisor.ack");
13495            let frame =
13496                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13497            let replies = handler.handle_supervisor_list(frame).await.unwrap();
13498            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13499            assert_eq!(
13500                body["modules"][0].get("configured_enabled"),
13501                configured.map(Value::Bool).as_ref()
13502            );
13503        }
13504    }
13505
13506    #[tokio::test(flavor = "current_thread")]
13507    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13508        let handler = ControlHandler::new(Arc::new(Registry::default()));
13509        let capture = EventCapture::default();
13510        let _subscriber =
13511            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13512        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13513        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13514
13515        let _ = handler
13516            .handle_client_control_request(
13517                &ctx,
13518                frame,
13519                ClientControlRequest::SupervisorSetEnabled {
13520                    module_id: "broca".to_string(),
13521                    enabled: false,
13522                },
13523                &mut None,
13524            )
13525            .await
13526            .unwrap();
13527
13528        let events = capture
13529            .events()
13530            .into_iter()
13531            .filter(|event| {
13532                event.target == "control"
13533                    && event.fields.get("message").map(String::as_str)
13534                        == Some("supervisor request received")
13535            })
13536            .collect::<Vec<_>>();
13537        assert_eq!(events.len(), 1, "one supervisor request log line");
13538        let event = &events[0];
13539        assert_eq!(event.level, tracing::Level::INFO);
13540        assert_eq!(
13541            event.fields.get("op"),
13542            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13543        );
13544        assert_eq!(
13545            event.fields.get("module_id"),
13546            Some(&"\"broca\"".to_string())
13547        );
13548        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13549        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13550        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13551    }
13552
13553    #[tokio::test(flavor = "current_thread")]
13554    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13555        let registry = Arc::new(Registry::default());
13556        let module_connection = ConnectionId::new(303);
13557        registry
13558            .register_with_control_ops(
13559                manifest("aft", PROTOCOL_VERSION),
13560                PROTOCOL_VERSION,
13561                module_connection,
13562                module_baseline_control_ops(),
13563            )
13564            .unwrap();
13565        let handler = ControlHandler::new(registry);
13566        let capture = EventCapture::default();
13567        let _subscriber =
13568            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13569        let (ctx, _rx) = route_ctx(module_connection);
13570        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13571
13572        let _ = handler
13573            .handle_client_control_request(
13574                &ctx,
13575                frame,
13576                ClientControlRequest::SupervisorSetEnabled {
13577                    module_id: "broca".to_string(),
13578                    enabled: false,
13579                },
13580                &mut None,
13581            )
13582            .await
13583            .unwrap();
13584
13585        let callers = capture
13586            .events()
13587            .into_iter()
13588            .filter(|event| {
13589                event.fields.get("message").map(String::as_str)
13590                    == Some("supervisor request received")
13591            })
13592            .map(|event| event.fields.get("caller").cloned())
13593            .collect::<Vec<_>>();
13594        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13595    }
13596
13597    #[tokio::test(flavor = "current_thread")]
13598    async fn supervisor_list_does_not_log_request_received() {
13599        let handler = ControlHandler::new(Arc::new(Registry::default()));
13600        let capture = EventCapture::default();
13601        let _subscriber =
13602            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13603        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13604        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13605
13606        handler
13607            .handle_client_control_request(
13608                &ctx,
13609                frame,
13610                ClientControlRequest::SupervisorList {},
13611                &mut None,
13612            )
13613            .await
13614            .unwrap();
13615
13616        assert!(capture.events().into_iter().all(|event| {
13617            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13618        }));
13619    }
13620
13621    #[tokio::test(flavor = "current_thread")]
13622    async fn supervisor_rescan_preview_does_not_log_request_received() {
13623        let handler = ControlHandler::new(Arc::new(Registry::default()));
13624        let capture = EventCapture::default();
13625        let _subscriber =
13626            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13627        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13628        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13629
13630        handler
13631            .handle_client_control_request(
13632                &ctx,
13633                frame,
13634                ClientControlRequest::SupervisorRescan { preview: true },
13635                &mut None,
13636            )
13637            .await
13638            .unwrap();
13639
13640        assert!(capture.events().into_iter().all(|event| {
13641            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13642        }));
13643    }
13644
13645    struct ApproveOperator;
13646    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13647        fn prompt(
13648            &self,
13649            _: &str,
13650            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13651        ) -> crate::operator_confirm::ProviderResult {
13652            crate::operator_confirm::ProviderResult::Approved
13653        }
13654    }
13655
13656    #[tokio::test]
13657    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13658        assert_eq!(
13659            ControlHandler::default().privileged_operator_prompt,
13660            PrivilegedOperatorPrompt::Log
13661        );
13662        for (initial, changed, running) in [
13663            ("log", "enforce", PrivilegedOperatorPrompt::Log),
13664            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13665        ] {
13666            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13667            let path = dir.join("subc.jsonc");
13668            std::fs::write(
13669                &path,
13670                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13671            )
13672            .unwrap();
13673            let config = crate::daemon_config::load(&path).unwrap().unwrap();
13674            let registry = Arc::new(Registry::default());
13675            let handle = SupervisorHandle::new();
13676            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13677                .with_handle(handle.clone());
13678            let handler = ControlHandler::new(registry)
13679                .with_supervisor(handle)
13680                .with_privileged_operator_prompt(config.privileged_operator_prompt)
13681                .with_supervisor_rescan(supervisor, &path, config.port);
13682            handler.forwarding.operator_confirms().configure(
13683                Arc::new(ApproveOperator),
13684                crate::operator_confirm::OperatorLimits::default(),
13685            );
13686            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13687
13688            for (disk_mode, needs_restart) in [
13689                (Some(initial), false),
13690                (Some(changed), true),
13691                (Some(changed), true),
13692                (None, running == PrivilegedOperatorPrompt::Enforce),
13693                (Some(initial), false),
13694            ] {
13695                let mut doc = json!({"version":1});
13696                if let Some(mode) = disk_mode {
13697                    doc["privileged_operator_prompt"] = json!(mode);
13698                }
13699                std::fs::write(&path, doc.to_string()).unwrap();
13700                for preview in [true, false] {
13701                    let request = ClientControlRequest::SupervisorRescan { preview };
13702                    let frame = Frame::build(
13703                        FrameType::Request,
13704                        control_flags(),
13705                        0,
13706                        0,
13707                        1,
13708                        serde_json::to_vec(&request).unwrap(),
13709                    )
13710                    .unwrap();
13711                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13712                    assert_eq!(frames.len(), 1);
13713                    let ClientControlResponse::SupervisorRescan { result } =
13714                        serde_json::from_slice(&frames[0].body).unwrap()
13715                    else {
13716                        panic!("expected a rescan response");
13717                    };
13718                    let expected = if needs_restart {
13719                        vec!["privileged_operator_prompt"]
13720                    } else {
13721                        Vec::new()
13722                    };
13723                    assert_eq!(result.restart_required, expected);
13724                    assert_eq!(handler.privileged_operator_prompt, running);
13725                    assert_eq!(
13726                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13727                        running
13728                    );
13729                }
13730            }
13731        }
13732    }
13733
13734    #[tokio::test]
13735    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13736        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13737        let path = dir.join("subc.jsonc");
13738        let registry = Arc::new(Registry::default());
13739        let handle = SupervisorHandle::new();
13740        let supervisor =
13741            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13742        let handler = ControlHandler::new(registry)
13743            .with_supervisor(handle)
13744            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13745            .with_supervisor_rescan(supervisor, &path, None);
13746        handler.forwarding.operator_confirms().configure(
13747            Arc::new(ApproveOperator),
13748            crate::operator_confirm::OperatorLimits::default(),
13749        );
13750        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13751        for preview in [true, false] {
13752            std::fs::write(
13753                &path,
13754                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13755            )
13756            .unwrap();
13757            let request = ClientControlRequest::SupervisorRescan { preview };
13758            let frame = Frame::build(
13759                FrameType::Request,
13760                control_flags(),
13761                0,
13762                0,
13763                1,
13764                serde_json::to_vec(&request).unwrap(),
13765            )
13766            .unwrap();
13767            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13768            assert_eq!(frames.len(), 1);
13769            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13770            assert_eq!(
13771                handler.privileged_operator_prompt,
13772                PrivilegedOperatorPrompt::Enforce
13773            );
13774        }
13775    }
13776
13777    /// One wire code has several senders, so the refusal line names the check
13778    /// that refused. This drives the shared refusal path for ordinary refusals
13779    /// with an unregistered
13780    /// target and requires the branch label on the event.
13781    #[tokio::test]
13782    async fn route_open_refusal_names_the_check_that_refused() {
13783        let handler = ControlHandler::new(Arc::new(Registry::default()));
13784        let capture = EventCapture::default();
13785        let _subscriber =
13786            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13787        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13788        let response = handler
13789            .handle_control_frame(
13790                &ctx,
13791                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13792            )
13793            .await
13794            .unwrap();
13795
13796        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13797        let event = capture
13798            .events()
13799            .into_iter()
13800            .find(|event| {
13801                event.target == "control"
13802                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13803            })
13804            .expect("route.open refusal event");
13805        assert_eq!(
13806            event.fields.get("reason"),
13807            Some(&"\"not_registered\"".to_string())
13808        );
13809    }
13810
13811    #[tokio::test]
13812    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13813        let registry = Arc::new(Registry::default());
13814        let supervisor_handle = SupervisorHandle::new();
13815        let supervisor =
13816            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13817                .with_handle(supervisor_handle.clone())
13818                .with_connection_file_path(std::env::temp_dir().join(format!(
13819                    "subc-route-open-refusal-info-{}",
13820                    std::process::id()
13821                )));
13822        let module = supervisor
13823            .supervise_configured(
13824                ModuleSpec {
13825                    module_id: "warming".to_string(),
13826                    program: fake_aft_stub_path(),
13827                    args: Vec::new(),
13828                    env: Vec::new(),
13829                    reserved: false,
13830                    reserved_prefixes: Vec::new(),
13831                    protocol: ModuleProtocol::Subc,
13832                    overlap: Default::default(),
13833                },
13834                true,
13835            )
13836            .unwrap();
13837        assert_eq!(module.state().unwrap(), ModuleState::Running);
13838
13839        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13840        assert!(handler
13841            .counters()
13842            .snapshot()
13843            .get("route_open_refused_by_code")
13844            .is_none());
13845        let capture = EventCapture::default();
13846        let _subscriber =
13847            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13848        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13849        let response = handler
13850            .handle_control_frame(
13851                &ctx,
13852                route_open_frame(394, "warming", unique_project_root("refusal-info")),
13853            )
13854            .await
13855            .unwrap();
13856        module.stop().await.unwrap();
13857
13858        assert_eq!(parse_error(&response[0])["code"], "module_warming");
13859        let event = capture
13860            .events()
13861            .into_iter()
13862            .find(|event| {
13863                event.target == "control"
13864                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13865            })
13866            .expect("route.open refusal event");
13867        assert_eq!(
13868            event.fields.get("module_id"),
13869            Some(&"\"warming\"".to_string())
13870        );
13871        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13872        assert_eq!(
13873            event.fields.get("reason"),
13874            Some(&"\"supervised_not_registered\"".to_string())
13875        );
13876        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13877        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13878        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13879        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13880        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13881        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13882        assert_eq!(
13883            handler.counters().snapshot()["route_open_refused_by_code"],
13884            json!({ "module_warming": 1 })
13885        );
13886    }
13887
13888    const OUTAGE_START: &str = "route.open refusing module: not serving";
13889    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13890
13891    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13892        capture
13893            .events()
13894            .into_iter()
13895            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13896            .collect()
13897    }
13898
13899    fn supervise_stub(
13900        registry: &Arc<Registry>,
13901        module_id: &str,
13902        enabled: bool,
13903    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13904        let supervisor_handle = SupervisorHandle::new();
13905        let supervisor =
13906            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13907                .with_handle(supervisor_handle.clone())
13908                .with_connection_file_path(std::env::temp_dir().join(format!(
13909                    "subc-route-outage-{module_id}-{}",
13910                    std::process::id()
13911                )));
13912        let module = supervisor
13913            .supervise_configured(
13914                ModuleSpec {
13915                    module_id: module_id.to_string(),
13916                    program: fake_aft_stub_path(),
13917                    args: Vec::new(),
13918                    env: Vec::new(),
13919                    reserved: false,
13920                    reserved_prefixes: Vec::new(),
13921                    protocol: ModuleProtocol::Subc,
13922                    overlap: Default::default(),
13923                },
13924                enabled,
13925            )
13926            .unwrap();
13927        (supervisor_handle, module)
13928    }
13929
13930    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13931        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13932            module_id: module_id.to_string(),
13933            drain_timeout_ms: Some(50),
13934        })
13935        .unwrap();
13936        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13937    }
13938
13939    /// Two handlers built over one forwarding table must share one outage
13940    /// tracker; separate trackers would each log their own opening line for
13941    /// the same outage.
13942    #[test]
13943    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13944        let registry = Arc::new(Registry::default());
13945        let forwarding = Arc::new(ForwardingTable::default());
13946        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13947        let second = ControlHandler::with_forwarding(registry, forwarding);
13948        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13949    }
13950
13951    /// A client can name any module id it likes. Refusing an unknown one,
13952    /// however often, must not create outage state or outage lines, or the
13953    /// tracker would be a memory sink any client could fill.
13954    #[tokio::test(flavor = "current_thread")]
13955    async fn route_open_unknown_module_refusals_add_no_outage_state() {
13956        let handler = ControlHandler::new(Arc::new(Registry::default()));
13957        let capture = EventCapture::default();
13958        let _subscriber =
13959            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13960        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13961        for corr in 0..8 {
13962            let response = handler
13963                .handle_control_frame(
13964                    &ctx,
13965                    route_open_frame(
13966                        380 + corr,
13967                        &format!("nobody-{corr}"),
13968                        unique_project_root("outage-unknown"),
13969                    ),
13970                )
13971                .await
13972                .unwrap();
13973            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13974        }
13975
13976        assert_eq!(handler.route_outages.tracked_module_count(), 0);
13977        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13978        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13979    }
13980
13981    /// Drives the refusal path end to end: a supervised module that served
13982    /// before and stopped being registered with no instruction to stop is a
13983    /// WARN, and the same module refused after an operator `supervisor.restart`
13984    /// is an INFO.
13985    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13986    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13987        let registry = Arc::new(Registry::default());
13988        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
13989        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13990        let capture = EventCapture::default();
13991        let _subscriber =
13992            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13993        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13994        // The stub never registers, so pretend it served once: otherwise every
13995        // refusal would fall in its startup window.
13996        handler.route_outages.record_accepted("outage-restart");
13997
13998        let response = handler
13999            .handle_control_frame(
14000                &ctx,
14001                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14002            )
14003            .await
14004            .unwrap();
14005        assert_eq!(response[0].header.ty, FrameType::Error);
14006        let starts = outage_lines(&capture, OUTAGE_START);
14007        assert_eq!(starts.len(), 1, "{starts:?}");
14008        assert_eq!(starts[0].level, tracing::Level::WARN);
14009        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14010        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14011        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14012        handler.route_outages.record_accepted("outage-restart");
14013        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14014
14015        let restart = handler
14016            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14017            .await
14018            .unwrap();
14019        assert_eq!(
14020            restart[0].header.ty,
14021            FrameType::Response,
14022            "{:?}",
14023            parse_error(&restart[0])
14024        );
14025        handler
14026            .handle_control_frame(
14027                &ctx,
14028                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14029            )
14030            .await
14031            .unwrap();
14032        module.stop().await.unwrap();
14033
14034        let starts = outage_lines(&capture, OUTAGE_START);
14035        assert_eq!(starts.len(), 2, "{starts:?}");
14036        assert_eq!(starts[1].level, tracing::Level::INFO);
14037        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14038    }
14039
14040    /// A restart refused before it touched the module (here: the module is
14041    /// disabled) must clear its operator mark, so the next real outage is
14042    /// still reported as a warning.
14043    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14044    async fn failed_operator_restart_leaves_no_operator_mark() {
14045        let registry = Arc::new(Registry::default());
14046        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
14047        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14048        let capture = EventCapture::default();
14049        let _subscriber =
14050            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14051        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14052        handler.route_outages.record_accepted("outage-disabled");
14053
14054        let restart = handler
14055            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14056            .await
14057            .unwrap();
14058        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14059        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14060
14061        handler
14062            .handle_control_frame(
14063                &ctx,
14064                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14065            )
14066            .await
14067            .unwrap();
14068        let starts = outage_lines(&capture, OUTAGE_START);
14069        assert_eq!(starts.len(), 1, "{starts:?}");
14070        assert_eq!(starts[0].level, tracing::Level::WARN);
14071    }
14072
14073    #[tokio::test(flavor = "current_thread")]
14074    async fn route_open_unknown_module_escapes_target_module_id() {
14075        let handler = ControlHandler::new(Arc::new(Registry::default()));
14076        let capture = EventCapture::default();
14077        let _subscriber =
14078            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14079        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14080        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14081        let response = handler
14082            .handle_control_frame(
14083                &ctx,
14084                route_open_frame(
14085                    395,
14086                    hostile_module_id,
14087                    unique_project_root("hostile-target-module-id"),
14088                ),
14089            )
14090            .await
14091            .unwrap();
14092
14093        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14094        let event = capture
14095            .events()
14096            .into_iter()
14097            .find(|event| {
14098                event.target == "control"
14099                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14100            })
14101            .expect("route.open unknown-module refusal event");
14102        let logged = event.fields.get("module_id").expect("module_id field");
14103        assert!(!logged.bytes().any(|byte| byte < 0x20));
14104        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14105    }
14106
14107    /// A route.open refusal is the daemon's own answer, so its ERROR carries
14108    /// the daemon-origin bit on the frame the client receives.
14109    #[tokio::test(flavor = "current_thread")]
14110    async fn route_open_refusal_carries_daemon_origin() {
14111        let handler = ControlHandler::new(Arc::new(Registry::default()));
14112        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14113        let response = handler
14114            .handle_control_frame(
14115                &ctx,
14116                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14117            )
14118            .await
14119            .unwrap();
14120
14121        assert_eq!(response.len(), 1);
14122        assert_eq!(response[0].header.ty, FrameType::Error);
14123        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14124        assert!(
14125            response[0].header.flags.is_daemon_origin(),
14126            "route.open refusal must carry daemon origin: {:?}",
14127            response[0].header.flags
14128        );
14129    }
14130
14131    /// The drain `module_reloading` refusal is daemon-synthesized for an
14132    /// endpoint that is reloading, so it carries the daemon-origin bit on the
14133    /// frame the client receives.
14134    #[tokio::test(flavor = "current_thread")]
14135    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14136        let registry = Arc::new(Registry::default());
14137        let forwarding = Arc::new(ForwardingTable::default());
14138        let handler =
14139            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14140        let module_connection = ConnectionId::new(95);
14141        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14142        hello_via_sink(
14143            &handler,
14144            &module_ctx,
14145            &mut module_rx,
14146            hello_frame("aft", PROTOCOL_VERSION, 395),
14147        )
14148        .await;
14149        forwarding
14150            .begin_module_drain("aft", RouteCloseReason::Restart)
14151            .unwrap()
14152            .expect("the module endpoint drains");
14153
14154        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14155        let response = handler
14156            .handle_control_frame(
14157                &client_ctx,
14158                route_open_frame(396, "aft", unique_project_root("drain-origin")),
14159            )
14160            .await
14161            .unwrap();
14162
14163        assert_eq!(response.len(), 1);
14164        assert_eq!(response[0].header.ty, FrameType::Error);
14165        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14166        assert!(
14167            response[0].header.flags.is_daemon_origin(),
14168            "drain module_reloading must carry daemon origin: {:?}",
14169            response[0].header.flags
14170        );
14171    }
14172
14173    /// A module that refuses a bind with a daemon code still cannot mint a
14174    /// daemon refusal: the relayed `module_reloading` body reaches the client
14175    /// with the daemon-origin bit clear.
14176    #[tokio::test(flavor = "current_thread")]
14177    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14178        let registry = Arc::new(Registry::default());
14179        let forwarding = Arc::new(ForwardingTable::default());
14180        let handler =
14181            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14182        let module_connection = ConnectionId::new(95);
14183        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14184        hello_via_sink(
14185            &handler,
14186            &module_ctx,
14187            &mut module_rx,
14188            hello_frame("aft", PROTOCOL_VERSION, 395),
14189        )
14190        .await;
14191
14192        let client_connection = ConnectionId::new(96);
14193        let (client_ctx, _client_rx) = route_ctx(client_connection);
14194        let (route_task, bind) = relay_route_open(
14195            &handler,
14196            client_connection,
14197            &client_ctx.egress,
14198            &mut module_rx,
14199            396,
14200            "aft",
14201            "spoofed-reloading",
14202        )
14203        .await;
14204        // The module sets the origin bit on its own rejection too: neither
14205        // the bit nor the daemon code may survive the relay as daemon-owned.
14206        let rejection = Frame::build(
14207            FrameType::Error,
14208            crate::router::daemon_error_flags(),
14209            0,
14210            0,
14211            bind.header.corr,
14212            serde_json::to_vec(&ErrorBody::new(
14213                "module_reloading",
14214                "module refused route.bind",
14215            ))
14216            .unwrap(),
14217        )
14218        .unwrap();
14219        handler
14220            .handle_control_frame(&module_ctx, rejection)
14221            .await
14222            .unwrap();
14223
14224        let response = route_task.await.unwrap();
14225        assert_eq!(response.len(), 1);
14226        assert_eq!(response[0].header.ty, FrameType::Error);
14227        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14228        assert!(
14229            !response[0].header.flags.is_daemon_origin(),
14230            "a module's own rejection must not carry daemon origin: {:?}",
14231            response[0].header.flags
14232        );
14233    }
14234
14235    #[tokio::test(flavor = "current_thread")]
14236    async fn route_open_module_rejection_uses_daemon_counter_key() {
14237        let registry = Arc::new(Registry::default());
14238        let forwarding = Arc::new(ForwardingTable::default());
14239        let handler =
14240            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14241        let module_connection = ConnectionId::new(95);
14242        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14243        hello_via_sink(
14244            &handler,
14245            &module_ctx,
14246            &mut module_rx,
14247            hello_frame("aft", PROTOCOL_VERSION, 395),
14248        )
14249        .await;
14250
14251        let client_connection = ConnectionId::new(96);
14252        let (client_ctx, _client_rx) = route_ctx(client_connection);
14253        let capture = EventCapture::default();
14254        let _subscriber =
14255            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14256        let (route_task, bind) = relay_route_open(
14257            &handler,
14258            client_connection,
14259            &client_ctx.egress,
14260            &mut module_rx,
14261            396,
14262            "aft",
14263            "hostile-module-code",
14264        )
14265        .await;
14266        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14267        let rejection = Frame::build(
14268            FrameType::Error,
14269            control_flags(),
14270            0,
14271            0,
14272            bind.header.corr,
14273            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14274        )
14275        .unwrap();
14276        handler
14277            .handle_control_frame(&module_ctx, rejection)
14278            .await
14279            .unwrap();
14280
14281        let response = route_task.await.unwrap();
14282        assert_eq!(parse_error(&response[0])["code"], hostile_code);
14283        let counters = handler.counters().snapshot();
14284        assert_eq!(
14285            counters["route_open_refused_by_code"],
14286            json!({ "module_rejected": 1 })
14287        );
14288        assert!(counters["route_open_refused_by_code"]
14289            .get(hostile_code)
14290            .is_none());
14291
14292        let event = capture
14293            .events()
14294            .into_iter()
14295            .find(|event| {
14296                event.target == "control"
14297                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14298            })
14299            .expect("route.open module-rejection refusal event");
14300        let logged = event.fields.get("module_code").expect("module_code field");
14301        assert!(!logged.bytes().any(|byte| byte < 0x20));
14302        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14303    }
14304
14305    #[tokio::test]
14306    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14307        let registry = Arc::new(Registry::default());
14308        let supervisor_handle = SupervisorHandle::new();
14309        let missing_program = std::env::temp_dir().join(format!(
14310            "subc-route-open-missing-program-{}",
14311            std::process::id()
14312        ));
14313        let supervisor =
14314            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14315                .with_handle(supervisor_handle.clone());
14316        let module = supervisor
14317            .supervise_configured(
14318                ModuleSpec {
14319                    module_id: "failed".to_string(),
14320                    program: missing_program,
14321                    args: Vec::new(),
14322                    env: Vec::new(),
14323                    reserved: false,
14324                    reserved_prefixes: Vec::new(),
14325                    protocol: ModuleProtocol::Subc,
14326                    overlap: Default::default(),
14327                },
14328                true,
14329            )
14330            .unwrap();
14331        assert_eq!(module.state().unwrap(), ModuleState::Failed);
14332
14333        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14334        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14335        let response = handler
14336            .handle_control_frame(
14337                &ctx,
14338                route_open_frame(305, "failed", unique_project_root("failed")),
14339            )
14340            .await
14341            .unwrap();
14342
14343        assert_eq!(response[0].header.ty, FrameType::Error);
14344        let error = parse_error(&response[0]);
14345        assert_eq!(error["code"], "target_unavailable");
14346        assert!(error["message"]
14347            .as_str()
14348            .unwrap()
14349            .contains("state=failed, enabled=true, live=false"));
14350    }
14351
14352    #[tokio::test]
14353    async fn route_open_role_mismatch_remains_target_unavailable() {
14354        let registry = Arc::new(Registry::default());
14355        let handler = ControlHandler::new(Arc::clone(&registry));
14356        handler
14357            .handle_control(
14358                ConnectionId::new(41),
14359                non_routable_hello_frame_with_control_ops("health-only", 306, None),
14360            )
14361            .unwrap();
14362
14363        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14364        let response = handler
14365            .handle_control_frame(
14366                &ctx,
14367                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14368            )
14369            .await
14370            .unwrap();
14371
14372        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14373        assert!(parse_error(&response[0])["message"]
14374            .as_str()
14375            .unwrap()
14376            .contains("does not provide the requested target"));
14377    }
14378
14379    #[tokio::test]
14380    async fn route_open_inactive_registration_remains_target_unavailable() {
14381        let registry = Arc::new(Registry::default());
14382        let handler = ControlHandler::new(Arc::clone(&registry));
14383        handler
14384            .handle_control(
14385                ConnectionId::new(43),
14386                hello_frame("inactive", PROTOCOL_VERSION, 308),
14387            )
14388            .unwrap();
14389        assert!(registry
14390            .set_module_state_for_test("inactive", ChannelState::Closed)
14391            .unwrap());
14392
14393        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14394        let response = handler
14395            .handle_control_frame(
14396                &ctx,
14397                route_open_frame(309, "inactive", unique_project_root("inactive")),
14398            )
14399            .await
14400            .unwrap();
14401
14402        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14403        assert!(parse_error(&response[0])["message"]
14404            .as_str()
14405            .unwrap()
14406            .contains("is not active"));
14407    }
14408
14409    #[tokio::test]
14410    async fn late_health_reply_is_recorded_through_the_module_response_path() {
14411        let registry = Arc::new(Registry::default());
14412        let forwarding = Arc::new(ForwardingTable::default());
14413        let supervisor_handle = SupervisorHandle::new();
14414        let supervisor =
14415            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
14416                .with_forwarding(Arc::clone(&forwarding))
14417                .with_handle(supervisor_handle.clone());
14418        let module = supervisor
14419            .supervise_configured(
14420                crate::ModuleSpec {
14421                    module_id: "late-health-response".to_string(),
14422                    program: PathBuf::from("disabled-module"),
14423                    args: Vec::new(),
14424                    env: Vec::new(),
14425                    reserved: false,
14426                    reserved_prefixes: Vec::new(),
14427                    protocol: ModuleProtocol::Subc,
14428                    overlap: Default::default(),
14429                },
14430                false,
14431            )
14432            .unwrap();
14433        let handler =
14434            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14435                .with_supervisor(supervisor_handle);
14436        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14437        handler
14438            .handle_control_frame(
14439                &module_ctx,
14440                hello_frame_with_control_ops(
14441                    "late-health-response",
14442                    PROTOCOL_VERSION,
14443                    7,
14444                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14445                ),
14446            )
14447            .await
14448            .unwrap();
14449        let probe_started_at = Instant::now() - Duration::from_millis(80);
14450        let pending = forwarding
14451            .begin_health_probe_rpc_for(
14452                "late-health-response",
14453                MODULE_CONTROL_OP_HEALTH_CHECK,
14454                probe_started_at,
14455                Instant::now() - Duration::from_millis(1),
14456            )
14457            .unwrap();
14458        assert!(forwarding
14459            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14460            .unwrap());
14461
14462        let responses = handler
14463            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14464            .await
14465            .unwrap();
14466
14467        assert!(responses.is_empty());
14468        let health = module.status().unwrap().health;
14469        assert_eq!(health.late_answer_count, 1);
14470        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14471    }
14472
14473    #[tokio::test]
14474    async fn health_probe_timeout_and_module_death_are_typed() {
14475        let registry = Arc::new(Registry::default());
14476        let forwarding = Arc::new(ForwardingTable::default());
14477        let handler =
14478            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14479                .with_health_probe_timeout(Duration::from_millis(50));
14480        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14481        hello_via_sink(
14482            &handler,
14483            &module_ctx,
14484            &mut module_rx,
14485            hello_frame_with_control_ops(
14486                "aft",
14487                PROTOCOL_VERSION,
14488                7,
14489                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14490            ),
14491        )
14492        .await;
14493
14494        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14495        let responses = handler
14496            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14497            .await
14498            .unwrap();
14499        assert_eq!(responses[0].header.ty, FrameType::Error);
14500        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14501        let _ = module_rx.try_recv();
14502
14503        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14504        let health_handler = handler.clone();
14505        let death_task = tokio::spawn(async move {
14506            health_handler
14507                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14508                .await
14509                .unwrap()
14510        });
14511        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14512            .await
14513            .unwrap()
14514            .unwrap();
14515        handler
14516            .cleanup_connection(module_ctx.connection_id)
14517            .unwrap();
14518        let responses = death_task.await.unwrap();
14519        assert_eq!(responses[0].header.ty, FrameType::Error);
14520        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14521    }
14522
14523    #[test]
14524    fn hello_requires_exact_protocol_version() {
14525        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14526            let registry = Arc::new(Registry::default());
14527            let handler = ControlHandler::new(Arc::clone(&registry));
14528            let responses = handler
14529                .handle_control(
14530                    ConnectionId::new(connection),
14531                    hello_frame("aft", offered, 9),
14532                )
14533                .unwrap();
14534
14535            assert_eq!(responses.len(), 1);
14536            assert_eq!(responses[0].header.ty, FrameType::Error);
14537            let error = parse_error(&responses[0]);
14538            assert_eq!(error["code"], "version_unsupported");
14539            assert!(registry.get_module("aft").unwrap().is_none());
14540            assert_eq!(registry.active_registration_count().unwrap(), 0);
14541        }
14542    }
14543
14544    #[test]
14545    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14546        let registry = Arc::new(Registry::default());
14547        let forwarding = Arc::new(ForwardingTable::default());
14548        let handler =
14549            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14550        let module_connection = ConnectionId::new(301);
14551        let registration = registry
14552            .register_with_control_ops(
14553                manifest("aft-push", PROTOCOL_VERSION),
14554                PROTOCOL_VERSION,
14555                module_connection,
14556                module_baseline_control_ops(),
14557            )
14558            .unwrap();
14559        let (module_tx, _module_rx) = mpsc::channel(8);
14560        let endpoint = forwarding
14561            .register_module_connection(
14562                module_connection,
14563                "aft-push".to_string(),
14564                PROTOCOL_VERSION,
14565                manifest_concurrency(&registration.manifest),
14566                FrameSink::new(module_tx),
14567            )
14568            .unwrap();
14569
14570        // A push op this version does not know is ignored (forward-compat), not errored.
14571        let unknown = Frame::build(
14572            FrameType::Push,
14573            control_flags(),
14574            0,
14575            0,
14576            5,
14577            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14578        )
14579        .unwrap();
14580        let out = handler.handle_status_update(endpoint, unknown).unwrap();
14581        assert!(
14582            out.is_empty(),
14583            "unknown push op must be ignored, got {out:?}"
14584        );
14585
14586        // A malformed body for a KNOWN op is a real error worth surfacing.
14587        let malformed = Frame::build(
14588            FrameType::Push,
14589            control_flags(),
14590            0,
14591            0,
14592            6,
14593            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14594        )
14595        .unwrap();
14596        let out = handler.handle_status_update(endpoint, malformed).unwrap();
14597        assert_eq!(out.len(), 1);
14598        assert_eq!(out[0].header.ty, FrameType::Error);
14599        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14600    }
14601
14602    #[test]
14603    fn hello_rejected_when_connection_already_owns_client_routes() {
14604        let registry = Arc::new(Registry::default());
14605        let forwarding = Arc::new(ForwardingTable::default());
14606        let handler =
14607            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14608        // Commits a client route on connection 202 (bound to a module on conn 101).
14609        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
14610        let client_connection = ConnectionId::new(202);
14611
14612        // That same connection now tries to register as a module: rejected, so one
14613        // connection never holds both client-route and module-endpoint state.
14614        let responses = handler
14615            .handle_control(
14616                client_connection,
14617                hello_frame("aft-second", PROTOCOL_VERSION, 9),
14618            )
14619            .unwrap();
14620        assert_eq!(responses[0].header.ty, FrameType::Error);
14621        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14622        assert!(registry.get_module("aft-second").unwrap().is_none());
14623    }
14624
14625    #[tokio::test]
14626    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14627        let registry = Arc::new(Registry::default());
14628        let forwarding = Arc::new(ForwardingTable::default());
14629        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14630        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14631        hello_via_sink(
14632            &handler,
14633            &module_ctx,
14634            &mut module_rx,
14635            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14636        )
14637        .await;
14638        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14639        let pending = forwarding
14640            .begin_route_bind_relay_for_test(
14641                client_ctx.connection_id,
14642                client_ctx.egress.clone(),
14643                2,
14644                "alpha",
14645            )
14646            .unwrap();
14647        forwarding
14648            .complete_pending_relay(
14649                module_ctx.connection_id,
14650                pending.corr,
14651                RouteBindRelayOutcome::Accepted,
14652            )
14653            .unwrap();
14654        client_rx.try_recv().unwrap();
14655        for module_id in ["beta", "alpha"] {
14656            let replies = handler
14657                .handle_control_frame(
14658                    &module_ctx,
14659                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14660                )
14661                .await
14662                .unwrap();
14663            assert_eq!(replies.len(), 1, "second HELLO must be refused");
14664            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14665        }
14666        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14667        assert!(registry.get_module("beta").unwrap().is_none());
14668        assert!(matches!(
14669            forwarding
14670                .lookup_data_route(
14671                    client_ctx.connection_id,
14672                    pending.client_channel,
14673                    pending.client_epoch,
14674                )
14675                .unwrap(),
14676            DataRoute::Client(DataRouteState::Bound(_))
14677        ));
14678        assert!(handler
14679            .hello_launch_nonces
14680            .lock()
14681            .unwrap()
14682            .presented(module_ctx.connection_id, Some("alpha-nonce")));
14683        assert!(module_rx.try_recv().is_err());
14684    }
14685
14686    #[test]
14687    fn reserved_module_hello_requires_matching_launch_nonce() {
14688        let registry = Arc::new(Registry::default());
14689        let supervisor = SupervisorHandle::new();
14690        // The supervisor recorded the nonce it injected when it spawned the reserved
14691        // module; the HELLO verifier checks against the same shared handle.
14692        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14693        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14694
14695        // A HELLO with NO nonce is rejected.
14696        let no_nonce = handler
14697            .handle_control(
14698                ConnectionId::new(1),
14699                hello_frame("vault", PROTOCOL_VERSION, 1),
14700            )
14701            .unwrap();
14702        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14703        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14704        assert!(registry.get_module("vault").unwrap().is_none());
14705
14706        // A HELLO with the WRONG nonce is rejected.
14707        let wrong = handler
14708            .handle_control(
14709                ConnectionId::new(2),
14710                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14711            )
14712            .unwrap();
14713        assert_eq!(wrong[0].header.ty, FrameType::Error);
14714        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14715        assert!(registry.get_module("vault").unwrap().is_none());
14716
14717        // A HELLO with the CORRECT nonce registers.
14718        let ok = handler
14719            .handle_control(
14720                ConnectionId::new(3),
14721                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14722            )
14723            .unwrap();
14724        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14725        assert!(registry.get_module("vault").unwrap().is_some());
14726    }
14727
14728    #[test]
14729    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14730        let registry = Arc::new(Registry::default());
14731        let supervisor = SupervisorHandle::new();
14732        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14733        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14734        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14735
14736        let squat = handler
14737            .handle_control(
14738                ConnectionId::new(1),
14739                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14740            )
14741            .unwrap();
14742        assert_eq!(squat[0].header.ty, FrameType::Error);
14743        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14744        assert!(parse_error(&squat[0])["message"]
14745            .as_str()
14746            .unwrap()
14747            .contains("fed:"));
14748
14749        let accepted_peer = handler
14750            .handle_control(
14751                ConnectionId::new(2),
14752                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14753            )
14754            .unwrap();
14755        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14756
14757        let accepted_short = handler
14758            .handle_control(
14759                ConnectionId::new(3),
14760                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14761            )
14762            .unwrap();
14763        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14764
14765        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14766            let response = handler
14767                .handle_control(
14768                    ConnectionId::new(conn),
14769                    hello_frame(module_id, PROTOCOL_VERSION, conn),
14770                )
14771                .unwrap();
14772            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14773        }
14774    }
14775
14776    #[test]
14777    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14778        let registry = Arc::new(Registry::default());
14779        let supervisor = SupervisorHandle::new();
14780        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14781        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14782        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14783        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
14784
14785        let owner_nonce = handler
14786            .handle_control(
14787                ConnectionId::new(1),
14788                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14789            )
14790            .unwrap();
14791        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14792        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14793        assert!(registry.get_module("fed:special").unwrap().is_none());
14794
14795        let exact_nonce = handler
14796            .handle_control(
14797                ConnectionId::new(2),
14798                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14799            )
14800            .unwrap();
14801        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14802        assert!(registry.get_module("fed:special").unwrap().is_some());
14803    }
14804
14805    #[test]
14806    fn non_reserved_module_ignores_launch_nonce() {
14807        let registry = Arc::new(Registry::default());
14808        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
14809        // registration succeeds whether a spawned process echoes a nonce or not.
14810        let handler = ControlHandler::new(Arc::clone(&registry));
14811        let no_nonce = handler
14812            .handle_control(
14813                ConnectionId::new(1),
14814                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14815            )
14816            .unwrap();
14817        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14818        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14819
14820        let echoed_nonce = handler
14821            .handle_control(
14822                ConnectionId::new(2),
14823                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14824            )
14825            .unwrap();
14826        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14827        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14828    }
14829
14830    #[test]
14831    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14832        let handler = ControlHandler::default();
14833        let conn = ConnectionId::new(1);
14834        let malformed = Frame::build(
14835            FrameType::Hello,
14836            control_flags(),
14837            0,
14838            0,
14839            3,
14840            b"{not json".to_vec(),
14841        )
14842        .unwrap();
14843
14844        let error = handler.handle_control(conn, malformed).unwrap();
14845        assert_eq!(error[0].header.ty, FrameType::Error);
14846        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14847
14848        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14849        let pong = handler.handle_control(conn, ping).unwrap();
14850        assert_eq!(pong[0].header.ty, FrameType::Pong);
14851        assert_eq!(pong[0].header.corr, 4);
14852    }
14853
14854    #[test]
14855    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14856        let registry = Arc::new(Registry::default());
14857        let handler = ControlHandler::new(Arc::clone(&registry));
14858
14859        handler
14860            .handle_control(
14861                ConnectionId::new(1),
14862                hello_frame("aft", PROTOCOL_VERSION, 1),
14863            )
14864            .unwrap();
14865        let duplicate = handler
14866            .handle_control(
14867                ConnectionId::new(2),
14868                hello_frame("aft", PROTOCOL_VERSION, 2),
14869            )
14870            .unwrap();
14871
14872        assert_eq!(duplicate[0].header.ty, FrameType::Error);
14873        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14874        let registration = registry.get_module("aft").unwrap().unwrap();
14875        assert_eq!(registration.connection_id, ConnectionId::new(1));
14876    }
14877
14878    #[test]
14879    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14880        let registry = Arc::new(Registry::default());
14881        let forwarding = Arc::new(ForwardingTable::default());
14882        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14883        let handler =
14884            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14885                .with_process_liveness(process_liveness);
14886        let (ctx, route_channel, route_epoch) =
14887            bind_liveness_route(&registry, &forwarding, "aft-dead");
14888        let responses = handler
14889            .handle_route_poll(
14890                &ctx,
14891                route_poll_frame(41, PollKind::Liveness, route_channel),
14892                route_channel,
14893                route_epoch,
14894                PollKind::Liveness,
14895            )
14896            .unwrap();
14897
14898        assert_eq!(responses.len(), 1);
14899        assert_eq!(responses[0].header.ty, FrameType::Response);
14900        assert_route_poll_liveness(&responses[0], false);
14901    }
14902
14903    #[test]
14904    fn liveness_poll_without_process_source_uses_bound_route() {
14905        let registry = Arc::new(Registry::default());
14906        let forwarding = Arc::new(ForwardingTable::default());
14907        let handler =
14908            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14909        let (ctx, route_channel, route_epoch) =
14910            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
14911        let responses = handler
14912            .handle_route_poll(
14913                &ctx,
14914                route_poll_frame(42, PollKind::Liveness, route_channel),
14915                route_channel,
14916                route_epoch,
14917                PollKind::Liveness,
14918            )
14919            .unwrap();
14920
14921        assert_route_poll_liveness(&responses[0], true);
14922    }
14923
14924    #[test]
14925    fn liveness_poll_untracked_process_source_uses_bound_route() {
14926        let registry = Arc::new(Registry::default());
14927        let forwarding = Arc::new(ForwardingTable::default());
14928        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14929        let handler =
14930            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14931                .with_process_liveness(process_liveness);
14932        let (ctx, route_channel, route_epoch) =
14933            bind_liveness_route(&registry, &forwarding, "aft-untracked");
14934        let responses = handler
14935            .handle_route_poll(
14936                &ctx,
14937                route_poll_frame(43, PollKind::Liveness, route_channel),
14938                route_channel,
14939                route_epoch,
14940                PollKind::Liveness,
14941            )
14942            .unwrap();
14943
14944        assert_route_poll_liveness(&responses[0], true);
14945    }
14946
14947    #[tokio::test]
14948    async fn unknown_op_returns_unknown_control_op() {
14949        let handler = ControlHandler::default();
14950        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14951        let request = Frame::build(
14952            FrameType::Request,
14953            control_flags(),
14954            0,
14955            0,
14956            55,
14957            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14958        )
14959        .unwrap();
14960
14961        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14962
14963        assert_eq!(response.len(), 1);
14964        assert_eq!(response[0].header.ty, FrameType::Error);
14965        assert_eq!(response[0].header.corr, 55);
14966        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14967    }
14968
14969    #[tokio::test]
14970    async fn supervisor_provenance_rejects_unknown_exact_module() {
14971        let handler = ControlHandler::default();
14972        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14973        let request = Frame::build(
14974            FrameType::Request,
14975            control_flags(),
14976            0,
14977            0,
14978            57,
14979            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14980        )
14981        .unwrap();
14982
14983        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14984
14985        assert_eq!(response.len(), 1);
14986        assert_eq!(response[0].header.ty, FrameType::Error);
14987        assert_eq!(response[0].header.corr, 57);
14988        let error = parse_error(&response[0]);
14989        assert_eq!(error["code"], "unknown_module");
14990        assert_eq!(error["message"], "module_id 'missing' is not supervised");
14991    }
14992
14993    #[test]
14994    fn provenance_probe_override_keeps_handler_tests_deterministic() {
14995        let expected = subc_control::RunningImageAgreement::Unavailable {
14996            reason: subc_control::RunningImageUnavailableReason::HashFailed,
14997        };
14998        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14999        assert_eq!(handler.provenance_probe_override, Some(expected));
15000    }
15001
15002    #[test]
15003    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15004        let verdict = reload_verdict(
15005            std::path::Path::new("/bin/new"),
15006            Some(std::path::Path::new("/bin/old")),
15007            subc_control::RunningImageAgreement::Unavailable {
15008                reason: subc_control::RunningImageUnavailableReason::HashFailed,
15009            },
15010        );
15011        assert!(matches!(
15012            verdict.path,
15013            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15014                if configured == std::path::Path::new("/bin/new")
15015                    && spawned_from == std::path::Path::new("/bin/old")
15016        ));
15017    }
15018
15019    #[test]
15020    fn launch_spec_without_running_digest_is_unavailable_not_match() {
15021        use subc_control::{
15022            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15023        };
15024        let configured = crate::supervise::ModuleSpec {
15025            module_id: "legacy".into(),
15026            program: "/module".into(),
15027            args: Vec::new(),
15028            env: Vec::new(),
15029            reserved: false,
15030            reserved_prefixes: Vec::new(),
15031            protocol: subc_control::ModuleProtocol::Subc,
15032            overlap: Default::default(),
15033        };
15034        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15035        assert_eq!(
15036            launch_spec_agreement(&configured, false, Some(&digest)),
15037            Agreement::Unavailable {
15038                reason: Reason::NotRunning
15039            }
15040        );
15041        assert_eq!(
15042            launch_spec_agreement(&configured, true, None),
15043            Agreement::Unavailable {
15044                reason: Reason::SpawnedSpecUnavailable
15045            }
15046        );
15047        assert_eq!(
15048            launch_spec_agreement(&configured, true, Some(&digest)),
15049            Agreement::Match
15050        );
15051    }
15052
15053    #[cfg(windows)]
15054    #[test]
15055    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15056        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15057        let configured = dir.join("module.exe");
15058        std::fs::write(&configured, b"executable path fixture").unwrap();
15059        let spawned = std::fs::canonicalize(&configured).unwrap();
15060        assert_ne!(
15061            configured, spawned,
15062            "fixture must exercise different path spellings"
15063        );
15064        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15065        let lower_case = configured.to_string_lossy().to_lowercase();
15066        for spelling in [
15067            &configured,
15068            Path::new(&forward_slashes),
15069            Path::new(&lower_case),
15070        ] {
15071            let verdict = reload_verdict(
15072                spelling,
15073                Some(&spawned),
15074                subc_control::RunningImageAgreement::Unavailable {
15075                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15076                },
15077            );
15078            assert_eq!(
15079                verdict.path,
15080                ReloadPathAgreement::Match,
15081                "configured {spelling:?}, spawned {spawned:?}"
15082            );
15083        }
15084    }
15085
15086    #[cfg(windows)]
15087    #[test]
15088    fn windows_reload_path_keeps_different_existing_program_pending() {
15089        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15090        let old = dir.join("old.exe");
15091        let new = dir.join("new.exe");
15092        std::fs::write(&old, b"same executable content").unwrap();
15093        std::fs::copy(&old, &new).unwrap();
15094        let spawned = std::fs::canonicalize(&old).unwrap();
15095        let image = subc_control::RunningImageAgreement::Unavailable {
15096            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15097        };
15098        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15099        assert!(
15100            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15101            if configured == new && spawned_from == spawned)
15102        );
15103        let missing = dir.join("missing.exe");
15104        assert!(matches!(
15105            reload_verdict(&missing, Some(&spawned), image).path,
15106            ReloadPathAgreement::Mismatch { .. }
15107        ));
15108    }
15109
15110    #[test]
15111    fn reload_verdict_detects_replaced_image_at_same_path() {
15112        let image = subc_control::RunningImageAgreement::Mismatch {
15113            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15114                digest: "old".into(),
15115            },
15116            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15117                digest: "new".into(),
15118            },
15119        };
15120        let verdict = reload_verdict(
15121            std::path::Path::new("/bin/same"),
15122            Some(std::path::Path::new("/bin/same")),
15123            image.clone(),
15124        );
15125        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15126        assert_eq!(verdict.image, image);
15127    }
15128
15129    #[test]
15130    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15131        let image = subc_control::RunningImageAgreement::Unavailable {
15132            reason: subc_control::RunningImageUnavailableReason::NotRunning,
15133        };
15134        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15135        assert_eq!(
15136            verdict.path,
15137            subc_control::ReloadPathAgreement::Unavailable {
15138                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15139            }
15140        );
15141        assert_eq!(verdict.image, image);
15142
15143        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15144            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15145        };
15146        let verdict = reload_verdict(
15147            std::path::Path::new("/bin/same"),
15148            Some(std::path::Path::new("/bin/same")),
15149            unconfirmed.clone(),
15150        );
15151        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15152        assert_eq!(verdict.image, unconfirmed);
15153    }
15154
15155    #[test]
15156    fn reload_verdict_preserves_each_image_unavailability_reason() {
15157        use subc_control::RunningImageUnavailableReason as Reason;
15158
15159        for reason in [
15160            Reason::NotRunning,
15161            Reason::UnsupportedPlatform,
15162            Reason::RunningExecutableUnreadable,
15163            Reason::SpawnedPathUnreadable,
15164            Reason::HashFailed,
15165            Reason::ProcessIdentityUnconfirmed,
15166            Reason::Unknown("future_probe_reason".to_string()),
15167        ] {
15168            let image = subc_control::RunningImageAgreement::Unavailable {
15169                reason: reason.clone(),
15170            };
15171            let verdict = reload_verdict(
15172                std::path::Path::new("/bin/same"),
15173                Some(std::path::Path::new("/bin/same")),
15174                image.clone(),
15175            );
15176            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15177            assert_eq!(verdict.image, image, "{reason:?}");
15178        }
15179    }
15180
15181    #[tokio::test]
15182    async fn malformed_control_bodies_return_invalid_control_body() {
15183        let handler = ControlHandler::default();
15184        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15185
15186        for (corr, body) in [
15187            (56, br#"{"route_channel":1}"#.as_slice()),
15188            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15189            (
15190                58,
15191                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15192            ),
15193        ] {
15194            let request = Frame::build(
15195                FrameType::Request,
15196                control_flags(),
15197                0,
15198                0,
15199                corr,
15200                body.to_vec(),
15201            )
15202            .unwrap();
15203            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15204
15205            assert_eq!(response.len(), 1);
15206            assert_eq!(response[0].header.ty, FrameType::Error);
15207            assert_eq!(response[0].header.corr, corr);
15208            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15209        }
15210    }
15211
15212    #[tokio::test]
15213    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15214        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15215        let registry = Arc::new(Registry::default());
15216        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15217        let router = Router::with_control_handler(Arc::clone(&control));
15218        let connection = router.begin_connection();
15219        let (ctx, mut rx) = route_ctx(connection.id());
15220
15221        router
15222            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15223            .await
15224            .unwrap();
15225        let response = rx.recv().await.unwrap();
15226        let ack = parse_ack(&response);
15227        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15228        let channel = 1;
15229
15230        let goodbye =
15231            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15232        router.route_for_connection(&ctx, goodbye).await.unwrap();
15233        assert!(rx.try_recv().is_err());
15234        assert!(registry.get_module("aft").unwrap().is_none());
15235
15236        router
15237            .route_for_connection(&ctx, channel_request(channel, 13))
15238            .await
15239            .unwrap();
15240        let error_frame = rx.recv().await.unwrap();
15241        assert_eq!(error_frame.header.ty, FrameType::Error);
15242        assert_eq!(error_frame.header.channel, channel);
15243        let captured = crate::router::test_log::captured_logs(&logs);
15244        assert_eq!(
15245            captured
15246                .lines()
15247                .filter(|line| {
15248                    line.contains("module_id=aft")
15249                        && line.contains("reason=explicit_goodbye")
15250                        && line.contains("module registration ended")
15251                })
15252                .count(),
15253            1,
15254            "unexpected GOODBYE registry log: {captured}"
15255        );
15256    }
15257
15258    #[tokio::test]
15259    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15260        let registry = Arc::new(Registry::default());
15261        let handler = ControlHandler::new(registry).with_capability_config(
15262            [("prov".to_string(), true), ("cons".to_string(), true)],
15263            BTreeMap::new(),
15264        );
15265        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15266        register_capability_manifest(
15267            &handler,
15268            &provider_ctx,
15269            &mut provider_rx,
15270            capability_manifest("prov", &["thing/v1"], &[]),
15271            1,
15272        )
15273        .await;
15274        let mut consumer = capability_manifest("cons", &[], &[]);
15275        consumer.capabilities.as_mut().unwrap().requires.push(
15276            subc_protocol::manifest::CapabilityRequirement {
15277                capability: "thing/v1".to_string(),
15278                need: subc_protocol::manifest::CapabilityNeed::Required,
15279            },
15280        );
15281        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15282        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15283        assert_eq!(
15284            handler.capability_evaluator.verdict("cons", "thing/v1"),
15285            Some(CapabilityVerdict::Provided)
15286        );
15287        let (mut client_rx, _) = open_route_for_capability_test(
15288            &handler,
15289            &provider_ctx,
15290            &mut provider_rx,
15291            703,
15292            3,
15293            "prov",
15294            None,
15295        )
15296        .await;
15297        let goodbye =
15298            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15299        handler
15300            .handle_control_frame(&provider_ctx, goodbye)
15301            .await
15302            .unwrap();
15303        assert_eq!(
15304            handler.capability_evaluator.verdict("cons", "thing/v1"),
15305            Some(CapabilityVerdict::NeverProvided)
15306        );
15307        let closed = client_rx
15308            .try_recv()
15309            .expect("GOODBYE pushes route.closed before route GOODBYE");
15310        assert!(
15311            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15312            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15313        );
15314        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15315        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15316    }
15317
15318    struct RouteClosedFixture {
15319        handler: ControlHandler,
15320        #[cfg(unix)]
15321        supervisor: Supervisor,
15322        module_ctx: RouteCtx,
15323        #[cfg(unix)]
15324        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15325        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15326        route: Arc<crate::forwarding::RouteBinding>,
15327    }
15328
15329    async fn route_closed_fixture() -> RouteClosedFixture {
15330        let registry = Arc::new(Registry::default());
15331        let handler = ControlHandler::new(Arc::clone(&registry));
15332        #[cfg(unix)]
15333        let supervisor = Supervisor::new(registry, RestartPolicy::default())
15334            .with_forwarding(Arc::clone(&handler.forwarding));
15335        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15336        register_capability_manifest(
15337            &handler,
15338            &module_ctx,
15339            &mut module_rx,
15340            capability_manifest("prov", &[], &[]),
15341            1,
15342        )
15343        .await;
15344        let (client_rx, bind) = open_route_for_capability_test(
15345            &handler,
15346            &module_ctx,
15347            &mut module_rx,
15348            702,
15349            2,
15350            "prov",
15351            None,
15352        )
15353        .await;
15354        let ModuleControlRequest::RouteBind {
15355            route_channel: channel,
15356            epoch: route_epoch,
15357            ..
15358        } = bind
15359        else {
15360            panic!("expected route.bind");
15361        };
15362        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15363            handler
15364                .forwarding
15365                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15366                .unwrap()
15367        else {
15368            panic!("the acknowledged route must be bound");
15369        };
15370        RouteClosedFixture {
15371            handler,
15372            #[cfg(unix)]
15373            supervisor,
15374            module_ctx,
15375            #[cfg(unix)]
15376            module_rx,
15377            client_rx,
15378            route,
15379        }
15380    }
15381
15382    impl RouteClosedFixture {
15383        #[cfg(unix)]
15384        async fn cut(&mut self) {
15385            let notices = async {
15386                let mut draining = self.module_rx.recv().await.unwrap();
15387                assert!(matches!(
15388                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15389                        &draining.body
15390                    )
15391                    .unwrap(),
15392                    subc_protocol::session::ModuleControlCommand::Draining {
15393                        reason: RouteCloseReason::Restart,
15394                        ..
15395                    }
15396                ));
15397                draining.flushed.take().unwrap().send(()).unwrap();
15398                let mut closing = self.client_rx.recv().await.unwrap();
15399                assert!(matches!(
15400                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15401                    ClientControlPush::RouteClosing {
15402                        reason: RouteCloseReason::Restart,
15403                        ..
15404                    }
15405                ));
15406                closing.flushed.take().unwrap().send(()).unwrap();
15407            };
15408            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15409            result.unwrap();
15410        }
15411
15412        fn disconnect(&mut self) -> ClientControlPush {
15413            self.handler
15414                .cleanup_connection(self.module_ctx.connection_id)
15415                .unwrap();
15416            let closed = self.client_rx.try_recv().unwrap();
15417            assert_eq!(closed.header.ty, FrameType::Push);
15418            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15419            assert!(matches!(
15420                &push,
15421                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15422                    if channels == &[self.route.client_channel]
15423            ));
15424            assert_eq!(
15425                self.client_rx.try_recv().unwrap().header.ty,
15426                FrameType::Goodbye
15427            );
15428            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15429            push
15430        }
15431    }
15432
15433    #[cfg(unix)]
15434    #[tokio::test(start_paused = true)]
15435    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15436        let mut fixture = route_closed_fixture().await;
15437        fixture.cut().await;
15438        assert!(matches!(
15439            fixture.disconnect(),
15440            ClientControlPush::RouteClosed {
15441                reason: RouteCloseReason::Restart,
15442                drained: true,
15443                ..
15444            }
15445        ));
15446    }
15447
15448    #[cfg(unix)]
15449    #[tokio::test(start_paused = true)]
15450    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15451        let mut fixture = route_closed_fixture().await;
15452        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15453        let started = tokio::time::Instant::now();
15454        fixture.cut().await;
15455        assert!(started.elapsed() >= Duration::from_secs(2));
15456        assert_eq!(fixture.route.flow.in_flight(), 1);
15457        assert!(matches!(
15458            fixture.disconnect(),
15459            ClientControlPush::RouteClosed {
15460                reason: RouteCloseReason::Restart,
15461                drained: false,
15462                ..
15463            }
15464        ));
15465    }
15466
15467    #[tokio::test]
15468    async fn module_connection_crash_reports_crash_and_not_drained() {
15469        let mut fixture = route_closed_fixture().await;
15470        // The server calls cleanup_connection on module EOF. Without a planned
15471        // daemon shutdown, that disconnect must remain a crash with no prior
15472        // route.closing notice, even when the route has no in-flight calls.
15473        assert!(matches!(
15474            fixture.disconnect(),
15475            ClientControlPush::RouteClosed {
15476                reason: RouteCloseReason::Crash,
15477                drained: false,
15478                ..
15479            }
15480        ));
15481    }
15482
15483    #[tokio::test]
15484    async fn dropping_router_connection_releases_registration() {
15485        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15486        let registry = Arc::new(Registry::default());
15487        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15488        let router = Router::with_control_handler(Arc::clone(&control));
15489        let connection = router.begin_connection();
15490        let connection_id = connection.id();
15491        let (ctx, mut rx) = route_ctx(connection_id);
15492
15493        router
15494            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15495            .await
15496            .unwrap();
15497        let response = rx.recv().await.unwrap();
15498        let ack = parse_ack(&response);
15499        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15500        assert!(registry.get_module("aft").unwrap().is_some());
15501
15502        drop(connection);
15503
15504        assert!(registry.get_module("aft").unwrap().is_none());
15505        assert_eq!(registry.active_registration_count().unwrap(), 0);
15506
15507        control
15508            .cleanup_connection(ConnectionId::new(u64::MAX))
15509            .unwrap();
15510        let captured = crate::router::test_log::captured_logs(&logs);
15511        println!("captured registry lifecycle logs:\n{captured}");
15512        let events: Vec<_> = captured
15513            .lines()
15514            .filter(|line| {
15515                line.contains("module registered module_id=aft ")
15516                    || line.contains("module registration ended")
15517            })
15518            .collect();
15519        assert_eq!(
15520            events.len(),
15521            2,
15522            "unexpected registry lifecycle logs: {captured}"
15523        );
15524        assert!(events[0].contains("module registered module_id=aft "));
15525        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15526        assert!(events[1].contains(&format!(
15527            "module_id=aft connection_id={}",
15528            connection_id.get()
15529        )));
15530        assert!(events[1].contains("reason=connection_closed"));
15531        assert!(events[1].contains("module registration ended"));
15532        assert_eq!(
15533            captured
15534                .lines()
15535                .filter(|line| line.contains("module registered module_id=aft "))
15536                .count(),
15537            1,
15538            "legacy registration admission line must appear once: {captured}"
15539        );
15540    }
15541
15542    #[tokio::test]
15543    async fn hello_registration_keeps_legacy_module_registered_line_once() {
15544        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15545        let registry = Arc::new(Registry::default());
15546        let control = ControlHandler::new(Arc::clone(&registry));
15547        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15548        hello_via_sink(
15549            &control,
15550            &ctx,
15551            &mut rx,
15552            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15553        )
15554        .await;
15555
15556        let captured = crate::router::test_log::captured_logs(&logs);
15557        let admissions: Vec<_> = captured
15558            .lines()
15559            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15560            .collect();
15561        assert_eq!(
15562            admissions.len(),
15563            1,
15564            "legacy admission line must remain exactly once: {captured}"
15565        );
15566        assert!(admissions[0].contains("routable_provider=true"));
15567        assert!(admissions[0].contains("connection_id=777"));
15568    }
15569
15570    fn capability_manifest(
15571        module_id: &str,
15572        provides: &[&str],
15573        must_never_reach: &[&str],
15574    ) -> ModuleManifest {
15575        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15576        manifest.capabilities = Some(CapabilityDeclarations {
15577            provides: provides
15578                .iter()
15579                .map(|capability| (*capability).to_string())
15580                .collect(),
15581            requires: Vec::new(),
15582            must_never_reach: must_never_reach
15583                .iter()
15584                .map(|capability| (*capability).to_string())
15585                .collect(),
15586        });
15587        manifest
15588    }
15589
15590    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15591        Frame::build(
15592            FrameType::Hello,
15593            control_flags(),
15594            0,
15595            0,
15596            corr,
15597            serde_json::to_vec(&ModuleHelloBody {
15598                protocol_ver: manifest.protocol_ver,
15599                manifest,
15600                control_ops: None,
15601                launch_nonce: None,
15602            })
15603            .expect("capability test HELLO serializes"),
15604        )
15605        .expect("capability test HELLO frame builds")
15606    }
15607
15608    fn catalog_update_with_capabilities_frame(
15609        corr: u64,
15610        capabilities: CapabilityDeclarations,
15611    ) -> Frame {
15612        Frame::build(
15613            FrameType::Request,
15614            control_flags(),
15615            0,
15616            0,
15617            corr,
15618            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15619                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15620                capabilities: Some(capabilities),
15621                ready: None,
15622            })
15623            .expect("capability catalog.update serializes"),
15624        )
15625        .expect("capability catalog.update frame builds")
15626    }
15627
15628    async fn register_capability_manifest(
15629        handler: &ControlHandler,
15630        ctx: &RouteCtx,
15631        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15632        manifest: ModuleManifest,
15633        corr: u64,
15634    ) {
15635        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15636    }
15637
15638    async fn open_route_for_capability_test(
15639        handler: &ControlHandler,
15640        target_ctx: &RouteCtx,
15641        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15642        client_connection_id: u64,
15643        corr: u64,
15644        target_module_id: &str,
15645        consumer_identity: Option<ConsumerIdentity>,
15646    ) -> (
15647        mpsc::Receiver<crate::router::OutboundFrame>,
15648        ModuleControlRequest,
15649    ) {
15650        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15651        let route_handler = handler.clone();
15652        let target_module_id = target_module_id.to_string();
15653        let route_task = tokio::spawn(async move {
15654            route_handler
15655                .handle_control_frame(
15656                    &client_ctx,
15657                    route_open_frame_with_admission_facts(
15658                        corr,
15659                        &target_module_id,
15660                        unique_project_root("admission-facts"),
15661                        consumer_identity,
15662                        None,
15663                    ),
15664                )
15665                .await
15666                .expect("capability test route.open succeeds")
15667        });
15668        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15669            .await
15670            .expect("capability test route.open must reach route.bind")
15671            .expect("target control receiver stays open");
15672        let bind_request: ModuleControlRequest =
15673            serde_json::from_slice(&bind.body).expect("route.bind decodes");
15674        handler
15675            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15676            .await
15677            .expect("capability test route.bind ACK succeeds");
15678        assert!(route_task.await.expect("route.open task joins").is_empty());
15679        let opened = client_rx
15680            .recv()
15681            .await
15682            .expect("successful route.open publishes a response");
15683        assert!(matches!(
15684            serde_json::from_slice::<ClientControlResponse>(&opened.body),
15685            Ok(ClientControlResponse::RouteOpen { .. })
15686        ));
15687        (client_rx, bind_request)
15688    }
15689
15690    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15691        assert_eq!(frame.header.ty, FrameType::Push);
15692        assert_eq!(frame.header.channel, 0);
15693        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15694            .expect("route.closed control push decodes");
15695        let ClientControlPush::RouteClosed { channels, .. } = &push else {
15696            panic!("expected route.closed");
15697        };
15698        assert_eq!(channels.len(), 1, "exactly one violating route closed");
15699        let closed_channel = channels[0];
15700        let channels = channels.clone();
15701        assert_eq!(
15702            push,
15703            ClientControlPush::RouteClosed {
15704                module_id: target_module_id.to_string(),
15705                channels,
15706                reason: RouteCloseReason::CapabilityDenied,
15707                drained: false,
15708                abandoned: 0,
15709                excluded_subscriptions: 0,
15710                terminal: Some(false),
15711            }
15712        );
15713        closed_channel
15714    }
15715
15716    #[tokio::test]
15717    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15718        let registry = Arc::new(Registry::default());
15719        let forwarding = Arc::new(ForwardingTable::default());
15720        let supervisor = SupervisorHandle::new();
15721        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15722        let handler =
15723            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15724                .with_supervisor(supervisor);
15725        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15726        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15727        register_capability_manifest(
15728            &handler,
15729            &target_ctx,
15730            &mut target_rx,
15731            capability_manifest("target", &["credentials-provider/v1"], &[]),
15732            1,
15733        )
15734        .await;
15735        register_capability_manifest(
15736            &handler,
15737            &opener_ctx,
15738            &mut opener_rx,
15739            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15740            2,
15741        )
15742        .await;
15743
15744        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15745        let replies = handler
15746            .handle_control_frame(
15747                &client_ctx,
15748                route_open_frame_with_admission_facts(
15749                    3,
15750                    "target",
15751                    unique_project_root("admission-facts"),
15752                    Some(ConsumerIdentity {
15753                        module_id: "opener".to_string(),
15754                        launch_nonce: "opener-nonce".to_string(),
15755                    }),
15756                    None,
15757                ),
15758            )
15759            .await
15760            .expect("denied route.open returns a typed frame");
15761        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15762        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15763        assert!(
15764            target_rx.try_recv().is_err(),
15765            "forbidden route.open must not relay route.bind"
15766        );
15767    }
15768
15769    #[tokio::test]
15770    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15771        let registry = Arc::new(Registry::default());
15772        let forwarding = Arc::new(ForwardingTable::default());
15773        let supervisor = SupervisorHandle::new();
15774        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15775        let handler =
15776            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15777                .with_supervisor(supervisor);
15778        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15779        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15780        register_capability_manifest(
15781            &handler,
15782            &target_ctx,
15783            &mut target_rx,
15784            capability_manifest("target", &["credentials-provider/v1"], &[]),
15785            1,
15786        )
15787        .await;
15788        register_capability_manifest(
15789            &handler,
15790            &old_opener_ctx,
15791            &mut old_opener_rx,
15792            capability_manifest("opener", &[], &[]),
15793            2,
15794        )
15795        .await;
15796        let (mut client_rx, _) = open_route_for_capability_test(
15797            &handler,
15798            &target_ctx,
15799            &mut target_rx,
15800            712,
15801            3,
15802            "target",
15803            Some(ConsumerIdentity {
15804                module_id: "opener".to_string(),
15805                launch_nonce: "opener-nonce".to_string(),
15806            }),
15807        )
15808        .await;
15809        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15810
15811        handler
15812            .cleanup_connection(old_opener_ctx.connection_id)
15813            .expect("old opener registration cleans up");
15814        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15815        register_capability_manifest(
15816            &handler,
15817            &new_opener_ctx,
15818            &mut new_opener_rx,
15819            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15820            4,
15821        )
15822        .await;
15823
15824        let closed_channel = assert_capability_denied_push(
15825            client_rx
15826                .try_recv()
15827                .expect("HELLO deny addition must emit route.closed")
15828                .frame,
15829            "target",
15830        );
15831        let goodbye = client_rx
15832            .try_recv()
15833            .expect("HELLO deny addition must send the client a route GOODBYE")
15834            .frame;
15835        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15836        assert_eq!(goodbye.header.channel, closed_channel);
15837        assert_ne!(goodbye.header.channel, 0);
15838        assert_ne!(goodbye.header.epoch, 0);
15839        assert!(client_rx.try_recv().is_err());
15840        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15841        assert!(matches!(
15842            target_rx.try_recv(),
15843            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15844        ));
15845    }
15846
15847    #[tokio::test]
15848    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15849        let registry = Arc::new(Registry::default());
15850        let forwarding = Arc::new(ForwardingTable::default());
15851        let supervisor = SupervisorHandle::new();
15852        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15853        let handler =
15854            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15855                .with_supervisor(supervisor);
15856        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15857        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15858        register_capability_manifest(
15859            &handler,
15860            &target_ctx,
15861            &mut target_rx,
15862            capability_manifest("target", &[], &[]),
15863            1,
15864        )
15865        .await;
15866        register_capability_manifest(
15867            &handler,
15868            &opener_ctx,
15869            &mut opener_rx,
15870            capability_manifest("opener", &[], &["credentials-provider/v1"]),
15871            2,
15872        )
15873        .await;
15874        let (mut client_rx, _) = open_route_for_capability_test(
15875            &handler,
15876            &target_ctx,
15877            &mut target_rx,
15878            722,
15879            3,
15880            "target",
15881            Some(ConsumerIdentity {
15882                module_id: "opener".to_string(),
15883                launch_nonce: "opener-nonce".to_string(),
15884            }),
15885        )
15886        .await;
15887        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15888
15889        let replies = handler
15890            .handle_control_frame(
15891                &target_ctx,
15892                catalog_update_with_capabilities_frame(
15893                    4,
15894                    CapabilityDeclarations {
15895                        provides: vec!["credentials-provider/v1".to_string()],
15896                        requires: Vec::new(),
15897                        must_never_reach: Vec::new(),
15898                    },
15899                ),
15900            )
15901            .await
15902            .expect("claim catalog.update succeeds");
15903        assert!(matches!(
15904            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15905            Ok(ModuleControlResponseToModule::CatalogUpdate {})
15906        ));
15907        let closed_channel = assert_capability_denied_push(
15908            client_rx
15909                .try_recv()
15910                .expect("claim addition must emit route.closed")
15911                .frame,
15912            "target",
15913        );
15914        let goodbye = client_rx
15915            .try_recv()
15916            .expect("claim addition must send the client a route GOODBYE")
15917            .frame;
15918        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15919        assert_eq!(goodbye.header.channel, closed_channel);
15920        assert_ne!(goodbye.header.channel, 0);
15921        assert_ne!(goodbye.header.epoch, 0);
15922        assert!(client_rx.try_recv().is_err());
15923        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15924        assert!(matches!(
15925            target_rx.try_recv(),
15926            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15927        ));
15928    }
15929
15930    #[tokio::test]
15931    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15932        let registry = Arc::new(Registry::default());
15933        let forwarding = Arc::new(ForwardingTable::default());
15934        let supervisor = SupervisorHandle::new();
15935        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15936        let handler =
15937            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15938                .with_supervisor(supervisor);
15939        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15940        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15941        register_capability_manifest(
15942            &handler,
15943            &target_ctx,
15944            &mut target_rx,
15945            capability_manifest("target", &["credentials-provider/v1"], &[]),
15946            1,
15947        )
15948        .await;
15949        register_capability_manifest(
15950            &handler,
15951            &opener_ctx,
15952            &mut opener_rx,
15953            capability_manifest("opener", &[], &[]),
15954            2,
15955        )
15956        .await;
15957        let (mut client_rx, _) = open_route_for_capability_test(
15958            &handler,
15959            &target_ctx,
15960            &mut target_rx,
15961            732,
15962            3,
15963            "target",
15964            Some(ConsumerIdentity {
15965                module_id: "opener".to_string(),
15966                launch_nonce: "opener-nonce".to_string(),
15967            }),
15968        )
15969        .await;
15970        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15971
15972        handler
15973            .handle_control_frame(
15974                &target_ctx,
15975                catalog_update_with_capabilities_frame(
15976                    4,
15977                    CapabilityDeclarations {
15978                        provides: Vec::new(),
15979                        requires: Vec::new(),
15980                        must_never_reach: Vec::new(),
15981                    },
15982                ),
15983            )
15984            .await
15985            .expect("claim removal catalog.update succeeds");
15986        assert_eq!(
15987            forwarding.active_binding_count().unwrap(),
15988            1,
15989            "removing an attested target claim must leave the route census unchanged"
15990        );
15991        assert!(
15992            client_rx.try_recv().is_err(),
15993            "claim removal must not emit route.closed or a client route GOODBYE"
15994        );
15995        assert!(
15996            target_rx.try_recv().is_err(),
15997            "claim removal must not send the target a route GOODBYE"
15998        );
15999    }
16000
16001    /// A direct client may open a route to a denied capability provider; this
16002    /// policy applies only to attested supervised module origins, not to direct clients.
16003    #[tokio::test]
16004    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16005        let registry = Arc::new(Registry::default());
16006        let forwarding = Arc::new(ForwardingTable::default());
16007        let supervisor = SupervisorHandle::new();
16008        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16009        let handler =
16010            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16011                .with_supervisor(supervisor);
16012        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16013        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16014        register_capability_manifest(
16015            &handler,
16016            &target_ctx,
16017            &mut target_rx,
16018            capability_manifest("target", &["credentials-provider/v1"], &[]),
16019            1,
16020        )
16021        .await;
16022        register_capability_manifest(
16023            &handler,
16024            &opener_ctx,
16025            &mut opener_rx,
16026            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16027            2,
16028        )
16029        .await;
16030
16031        let (_client_rx, bind) = open_route_for_capability_test(
16032            &handler,
16033            &target_ctx,
16034            &mut target_rx,
16035            742,
16036            3,
16037            "target",
16038            None,
16039        )
16040        .await;
16041        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16042            panic!("direct scope-honesty route must bind");
16043        };
16044        assert_eq!(principal, Some(Principal::Direct));
16045        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16046    }
16047
16048    /// A module that denies a capability receives no self-route exemption when it
16049    /// also attestedly provides that capability.
16050    #[tokio::test]
16051    async fn must_never_reach_self_route_is_capability_forbidden() {
16052        let registry = Arc::new(Registry::default());
16053        let forwarding = Arc::new(ForwardingTable::default());
16054        let supervisor = SupervisorHandle::new();
16055        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16056        let handler =
16057            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16058                .with_supervisor(supervisor);
16059        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16060        register_capability_manifest(
16061            &handler,
16062            &self_ctx,
16063            &mut self_rx,
16064            capability_manifest(
16065                "self-provider",
16066                &["credentials-provider/v1"],
16067                &["credentials-provider/v1"],
16068            ),
16069            1,
16070        )
16071        .await;
16072
16073        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16074        let replies = handler
16075            .handle_control_frame(
16076                &client_ctx,
16077                route_open_frame_with_admission_facts(
16078                    2,
16079                    "self-provider",
16080                    unique_project_root("admission-facts"),
16081                    Some(ConsumerIdentity {
16082                        module_id: "self-provider".to_string(),
16083                        launch_nonce: "self-nonce".to_string(),
16084                    }),
16085                    None,
16086                ),
16087            )
16088            .await
16089            .expect("self-route refusal returns a typed frame");
16090        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16091        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16092        assert!(
16093            self_rx.try_recv().is_err(),
16094            "self denial must not relay route.bind"
16095        );
16096    }
16097
16098    #[test]
16099    fn unsupported_channel_zero_frame_returns_error() {
16100        let handler = ControlHandler::default();
16101        let request = Frame::build(
16102            FrameType::Request,
16103            control_flags(),
16104            0,
16105            0,
16106            21,
16107            b"opaque".to_vec(),
16108        )
16109        .unwrap();
16110
16111        let response = handler
16112            .handle_control(ConnectionId::new(1), request)
16113            .unwrap();
16114
16115        assert_eq!(response[0].header.ty, FrameType::Error);
16116        assert_eq!(
16117            parse_error(&response[0])["code"],
16118            "unsupported_control_frame"
16119        );
16120    }
16121
16122    /// Blue/green swap at the control-plane boundary. The supervisor that opens
16123    /// a swap is not wired yet, so the candidate is registered here directly
16124    /// into the registry and forwarding candidate slots, the way the swap's
16125    /// HELLO admission will.
16126    mod swap {
16127        use super::*;
16128
16129        const INCUMBENT: ConnectionId = ConnectionId::new(30);
16130        const CANDIDATE: ConnectionId = ConnectionId::new(40);
16131
16132        struct Swap {
16133            registry: Arc<Registry>,
16134            forwarding: Arc<ForwardingTable>,
16135            handler: ControlHandler,
16136            incumbent_ctx: RouteCtx,
16137            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16138            candidate_ctx: RouteCtx,
16139            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16140        }
16141
16142        async fn swap_with_incumbent() -> Swap {
16143            let registry = Arc::new(Registry::default());
16144            let forwarding = Arc::new(ForwardingTable::default());
16145            let handler =
16146                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
16147            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16148            hello_via_sink(
16149                &handler,
16150                &incumbent_ctx,
16151                &mut incumbent_rx,
16152                hello_frame("aft", PROTOCOL_VERSION, 7),
16153            )
16154            .await;
16155            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16156            Swap {
16157                registry,
16158                forwarding,
16159                handler,
16160                incumbent_ctx,
16161                incumbent_rx,
16162                candidate_ctx,
16163                candidate_rx,
16164            }
16165        }
16166
16167        fn register_candidate(swap: &Swap, ready: Option<bool>) {
16168            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16169            candidate_manifest.ready = ready;
16170            let registration = swap
16171                .registry
16172                .register_candidate_with_control_ops(
16173                    candidate_manifest,
16174                    PROTOCOL_VERSION,
16175                    CANDIDATE,
16176                    module_baseline_control_ops(),
16177                )
16178                .unwrap();
16179            swap.forwarding
16180                .register_candidate_module_connection(
16181                    CANDIDATE,
16182                    "aft".to_string(),
16183                    PROTOCOL_VERSION,
16184                    manifest_concurrency(&registration.manifest),
16185                    swap.candidate_ctx.egress.clone(),
16186                )
16187                .unwrap();
16188        }
16189
16190        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16191            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16192            swap.registry.promote_candidate("aft").unwrap().unwrap();
16193            cutover.incumbent.unwrap()
16194        }
16195
16196        /// A swap candidate's HELLO reserves its candidate registry entry
16197        /// before installing its candidate forwarding endpoint. The swap
16198        /// supervisor promotes a ready candidate as soon as it reads it as
16199        /// registered, and promotion cuts forwarding over to the candidate
16200        /// endpoint, so the lookup the supervisor reads
16201        /// (`Registry::published_registration`) must not return the candidate
16202        /// until that endpoint exists. Otherwise the cutover finds no candidate
16203        /// endpoint and the swap fails as if the candidate had disconnected.
16204        #[tokio::test]
16205        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16206            let mut swap = swap_with_incumbent().await;
16207            let hold = Arc::new(std::sync::Barrier::new(2));
16208            let handler = swap
16209                .handler
16210                .clone()
16211                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16212            handler
16213                .supervisor
16214                .open_swap("aft", "candidate-nonce".to_string());
16215            let hello = {
16216                let handler = handler.clone();
16217                let candidate_ctx = swap.candidate_ctx.clone();
16218                std::thread::spawn(move || {
16219                    handler
16220                        .handle_hello(
16221                            candidate_ctx.connection_id,
16222                            Some(candidate_ctx.egress.clone()),
16223                            hello_frame_with_nonce(
16224                                "aft",
16225                                PROTOCOL_VERSION,
16226                                8,
16227                                Some("candidate-nonce"),
16228                            ),
16229                        )
16230                        .unwrap()
16231                })
16232            };
16233
16234            hold.wait();
16235            // Parked: the candidate slot is claimed but not published, and
16236            // forwarding has no candidate endpoint.
16237            assert!(swap
16238                .registry
16239                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16240                .unwrap()
16241                .is_some());
16242            assert!(swap
16243                .registry
16244                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16245                .unwrap()
16246                .is_none());
16247            assert!(swap
16248                .forwarding
16249                .module_endpoint_for_connection(CANDIDATE)
16250                .unwrap()
16251                .is_none());
16252
16253            hold.wait();
16254            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16255            assert_eq!(
16256                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16257                FrameType::HelloAck
16258            );
16259            let published = swap
16260                .registry
16261                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16262                .unwrap()
16263                .expect("candidate is published once its HELLO completes");
16264            assert_eq!(published.connection_id, CANDIDATE);
16265            assert!(swap
16266                .forwarding
16267                .module_endpoint_for_connection(CANDIDATE)
16268                .unwrap()
16269                .is_some());
16270        }
16271
16272        fn keyed_total(counters: &Value, key: &str) -> u64 {
16273            counters[key]
16274                .as_object()
16275                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16276                .unwrap_or(0)
16277        }
16278
16279        #[tokio::test]
16280        async fn replacement_logs_old_end_and_new_admission_once() {
16281            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16282            let mut swap = swap_with_incumbent().await;
16283            swap.handler
16284                .supervisor
16285                .open_swap("aft", "candidate-nonce".to_string());
16286            hello_via_sink(
16287                &swap.handler,
16288                &swap.candidate_ctx,
16289                &mut swap.candidate_rx,
16290                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16291            )
16292            .await;
16293            cutover(&swap);
16294            swap.handler.cleanup_connection(INCUMBENT).unwrap();
16295
16296            let captured = crate::router::test_log::captured_logs(&logs);
16297            println!("captured replacement registry logs:\n{captured}");
16298            let new_admissions: Vec<_> = captured
16299                .lines()
16300                .filter(|line| {
16301                    line.contains("connection_id=40")
16302                        && line.contains("swap candidate registered; not routable until cutover")
16303                })
16304                .collect();
16305            assert_eq!(
16306                new_admissions.len(),
16307                1,
16308                "unexpected admission logs: {captured}"
16309            );
16310            assert!(new_admissions[0].contains("module_id=aft"));
16311            assert!(new_admissions[0].contains("connection_id=40"));
16312            assert!(
16313                new_admissions[0].contains("swap candidate registered; not routable until cutover")
16314            );
16315            assert!(new_admissions[0].contains("ready=true"));
16316
16317            let old_admissions: Vec<_> = captured
16318                .lines()
16319                .filter(|line| {
16320                    line.contains("module registered module_id=aft ")
16321                        && line.contains("connection_id=30")
16322                })
16323                .collect();
16324            assert_eq!(
16325                old_admissions.len(),
16326                1,
16327                "unexpected incumbent admission: {captured}"
16328            );
16329
16330            let old_ends: Vec<_> = captured
16331                .lines()
16332                .filter(|line| {
16333                    line.contains("connection_id=30") && line.contains("module registration ended")
16334                })
16335                .collect();
16336            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16337            assert!(old_ends[0].contains("module_id=aft"));
16338            assert!(old_ends[0].contains("connection_id=30"));
16339            assert!(old_ends[0].contains("reason=replaced"));
16340            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16341            assert_eq!(
16342                captured
16343                    .lines()
16344                    .filter(|line| line.contains("module registration promoted"))
16345                    .count(),
16346                1,
16347                "unexpected promotion logs: {captured}"
16348            );
16349        }
16350
16351        /// An ack from the incumbent for a bind it was sent before cutover,
16352        /// arriving before the incumbent is drained. The incumbent is the live
16353        /// connection carrying every other client's routes, so the ack must
16354        /// not end it: the waiting client is told to retry, the reservation is
16355        /// given back, and the incumbent is told to drop just that binding.
16356        #[tokio::test]
16357        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16358            let mut swap = swap_with_incumbent().await;
16359            let handler = swap.handler.clone();
16360
16361            // A co-tenant route, bound on the incumbent before the swap.
16362            let cotenant = ConnectionId::new(31);
16363            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16364            let (cotenant_task, cotenant_bind) = relay_route_open(
16365                &handler,
16366                cotenant,
16367                &cotenant_ctx.egress,
16368                &mut swap.incumbent_rx,
16369                100,
16370                "aft",
16371                "swap-cotenant",
16372            )
16373            .await;
16374            handler
16375                .handle_control_frame(
16376                    &swap.incumbent_ctx,
16377                    route_bind_ack(cotenant_bind.header.corr),
16378                )
16379                .await
16380                .unwrap();
16381            assert!(cotenant_task.await.unwrap().is_empty());
16382            let (cotenant_channel, cotenant_epoch) =
16383                published_route(&cotenant_rx.recv().await.unwrap());
16384
16385            // A second route.open, relayed to the incumbent and not yet acked.
16386            let caller = ConnectionId::new(32);
16387            let (caller_ctx, mut caller_rx) = route_ctx(caller);
16388            let (caller_task, caller_bind) = relay_route_open(
16389                &handler,
16390                caller,
16391                &caller_ctx.egress,
16392                &mut swap.incumbent_rx,
16393                101,
16394                "aft",
16395                "swap-caller",
16396            )
16397            .await;
16398            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16399
16400            register_candidate(&swap, None);
16401            cutover(&swap);
16402
16403            // The incumbent acks after promotion and before any drain.
16404            let ack = handler
16405                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16406                .await;
16407            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16408            if module_loop_error.is_some() {
16409                // What the connection loop does with an untranslated router
16410                // error: end the connection, releasing every route on it.
16411                handler.cleanup_connection(INCUMBENT).unwrap();
16412            }
16413
16414            // 1. The incumbent's other routes survive.
16415            assert!(
16416                cotenant_rx.try_recv().is_err(),
16417                "the co-tenant route on the incumbent was torn down by one late ack: \
16418                 {module_loop_error:?}"
16419            );
16420            assert!(matches!(
16421                swap.forwarding
16422                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16423                    .unwrap(),
16424                DataRoute::Client(DataRouteState::Bound(_))
16425            ));
16426            assert_eq!(module_loop_error, None);
16427            assert!(swap
16428                .registry
16429                .get_module_by_connection(INCUMBENT)
16430                .unwrap()
16431                .is_some());
16432
16433            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
16434            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16435                .await
16436                .expect("the incumbent is told to drop the abandoned binding")
16437                .unwrap()
16438                .frame;
16439            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16440            assert_eq!(goodbye.header.channel, abandoned_channel);
16441            assert_eq!(goodbye.header.epoch, abandoned_epoch);
16442            assert!(swap.incumbent_rx.try_recv().is_err());
16443
16444            // 3. The waiting client gets a retryable refusal and no route.
16445            let response = caller_task.await.unwrap();
16446            assert_eq!(response.len(), 1);
16447            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16448            assert!(caller_rx.try_recv().is_err());
16449
16450            // 4. The reservation pair is given back, and the pending bind
16451            //    settled exactly once: one accepted open (the co-tenant) and one
16452            //    refused open (the caller), nothing counted twice.
16453            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16454            let counters = handler.counters().snapshot();
16455            assert_eq!(
16456                keyed_total(&counters, "route_open_accepted_by_principal"),
16457                1
16458            );
16459            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16460            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16461        }
16462
16463        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
16464        /// id would resolve to the promoted candidate and every new route.open
16465        /// would be refused as reloading, leaving neither process routable.
16466        #[tokio::test]
16467        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16468            let mut swap = swap_with_incumbent().await;
16469            register_candidate(&swap, None);
16470            let incumbent = cutover(&swap);
16471            swap.forwarding
16472                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16473                .unwrap()
16474                .expect("the incumbent is still registered");
16475
16476            let client = ConnectionId::new(33);
16477            let (client_ctx, mut client_rx) = route_ctx(client);
16478            let route_handler = swap.handler.clone();
16479            let open_ctx = RouteCtx {
16480                connection_id: client,
16481                egress: client_ctx.egress.clone(),
16482            };
16483            let mut route_task = tokio::spawn(async move {
16484                route_handler
16485                    .handle_control_frame(
16486                        &open_ctx,
16487                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16488                    )
16489                    .await
16490                    .unwrap()
16491            });
16492            let bind = tokio::select! {
16493                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16494                response = &mut route_task => {
16495                    let response = response.unwrap();
16496                    panic!(
16497                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
16498                        parse_error(&response[0])["code"]
16499                    );
16500                }
16501            };
16502            swap.handler
16503                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16504                .await
16505                .unwrap();
16506            assert!(route_task.await.unwrap().is_empty());
16507            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16508            match swap
16509                .forwarding
16510                .lookup_data_route(client, channel, epoch)
16511                .unwrap()
16512            {
16513                DataRoute::Client(DataRouteState::Bound(route)) => {
16514                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16515                }
16516                other => panic!("expected a bound route on the candidate, got {other:?}"),
16517            }
16518            assert!(swap.incumbent_rx.try_recv().is_err());
16519        }
16520
16521        /// A candidate declares itself ready with `catalog.update` on its own
16522        /// connection. If the connection-keyed registry lookups searched only the
16523        /// active slot, this would answer `not_registered` and the candidate
16524        /// would never become ready.
16525        #[tokio::test]
16526        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16527            let swap = swap_with_incumbent().await;
16528            register_candidate(&swap, Some(false));
16529            let update = Frame::build(
16530                FrameType::Request,
16531                control_flags(),
16532                0,
16533                0,
16534                55,
16535                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16536                    provides: manifest("aft", PROTOCOL_VERSION).provides,
16537                    capabilities: None,
16538                    ready: Some(true),
16539                })
16540                .unwrap(),
16541            )
16542            .unwrap();
16543
16544            let replies = swap
16545                .handler
16546                .handle_control_frame(&swap.candidate_ctx, update)
16547                .await
16548                .unwrap();
16549
16550            assert_eq!(replies.len(), 1);
16551            assert_eq!(
16552                replies[0].header.ty,
16553                FrameType::Response,
16554                "candidate catalog.update was refused: {:?}",
16555                serde_json::from_slice::<Value>(&replies[0].body).ok()
16556            );
16557            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16558            assert_eq!(
16559                swap.registry
16560                    .get_module("aft")
16561                    .unwrap()
16562                    .unwrap()
16563                    .connection_id,
16564                INCUMBENT
16565            );
16566        }
16567    }
16568
16569    /// The HELLO gate while the supervisor has a swap open: only the nonce it
16570    /// minted for the candidate admits a second process, into the candidate
16571    /// slot, and that check runs ahead of the reserved-module gate.
16572    mod swap_admission {
16573        use super::*;
16574
16575        const INCUMBENT_NONCE: &str = "incumbent-nonce";
16576        const CANDIDATE_NONCE: &str = "candidate-nonce";
16577
16578        fn handler_with_incumbent(
16579            module_id: &str,
16580            reserved: bool,
16581        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16582            let registry = Arc::new(Registry::default());
16583            let supervisor = SupervisorHandle::new();
16584            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16585            if reserved {
16586                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16587            }
16588            let handler =
16589                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
16590            let incumbent = handler
16591                .handle_control(
16592                    ConnectionId::new(1),
16593                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16594                )
16595                .unwrap();
16596            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16597            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16598            (registry, supervisor, handler)
16599        }
16600
16601        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
16602        /// admits every nonce, so while a swap is open the swap gate is the only
16603        /// thing between a key-holder and the candidate slot. A nonce the
16604        /// supervisor did not mint, or none at all, is refused, and neither the
16605        /// incumbent's registration nor the candidate slot moves.
16606        #[test]
16607        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16608            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16609
16610            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16611                let replies = handler
16612                    .handle_control(
16613                        ConnectionId::new(connection),
16614                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16615                    )
16616                    .unwrap();
16617                assert_eq!(replies[0].header.ty, FrameType::Error);
16618                assert_eq!(
16619                    parse_error(&replies[0])["code"],
16620                    "swap_token_invalid",
16621                    "nonce {nonce:?}"
16622                );
16623            }
16624            assert!(registry.get_candidate("aft").unwrap().is_none());
16625            assert_eq!(
16626                registry.get_module("aft").unwrap().unwrap().connection_id,
16627                ConnectionId::new(1)
16628            );
16629
16630            // Control: the minted token is admitted, into the candidate slot,
16631            // and only once.
16632            let admitted = handler
16633                .handle_control(
16634                    ConnectionId::new(4),
16635                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16636                )
16637                .unwrap();
16638            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16639            assert_eq!(
16640                registry
16641                    .get_candidate("aft")
16642                    .unwrap()
16643                    .unwrap()
16644                    .connection_id,
16645                ConnectionId::new(4)
16646            );
16647            assert_eq!(
16648                registry.get_module("aft").unwrap().unwrap().connection_id,
16649                ConnectionId::new(1),
16650                "the candidate must not take the active slot"
16651            );
16652            let replayed = handler
16653                .handle_control(
16654                    ConnectionId::new(5),
16655                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16656                )
16657                .unwrap();
16658            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16659
16660            // The case only this gate covers: the incumbent has died mid-swap,
16661            // so its duplicate refusal is gone too, and without the gate a
16662            // key-holder would take the id's ACTIVE slot.
16663            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16664            let squatter = handler
16665                .handle_control(
16666                    ConnectionId::new(6),
16667                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16668                )
16669                .unwrap();
16670            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16671            assert!(
16672                registry.get_module("aft").unwrap().is_none(),
16673                "a squatter took the active slot of an id being swapped"
16674            );
16675        }
16676
16677        /// Design mutation arm (iii). A reserved module's candidate presents a
16678        /// nonce the reserved gate has never seen (that gate holds the
16679        /// incumbent's), so the swap gate must run first or the candidate is
16680        /// refused `reserved_module` and a reserved module can never be swapped.
16681        #[test]
16682        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16683            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16684
16685            let replies = handler
16686                .handle_control(
16687                    ConnectionId::new(2),
16688                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16689                )
16690                .unwrap();
16691
16692            assert_eq!(
16693                replies[0].header.ty,
16694                FrameType::HelloAck,
16695                "reserved candidate refused: {:?}",
16696                serde_json::from_slice::<Value>(&replies[0].body).ok()
16697            );
16698            assert_eq!(
16699                registry
16700                    .get_candidate("vault")
16701                    .unwrap()
16702                    .unwrap()
16703                    .connection_id,
16704                ConnectionId::new(2)
16705            );
16706        }
16707
16708        /// With no swap open the gate is inert: the incumbent's reserved gate
16709        /// and duplicate refusal behave exactly as before.
16710        #[test]
16711        fn without_an_open_swap_the_ordinary_gates_decide() {
16712            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16713            supervisor.close_swap("vault");
16714
16715            let candidate = handler
16716                .handle_control(
16717                    ConnectionId::new(2),
16718                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16719                )
16720                .unwrap();
16721            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16722            let duplicate = handler
16723                .handle_control(
16724                    ConnectionId::new(3),
16725                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16726                )
16727                .unwrap();
16728            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16729            assert!(registry.get_candidate("vault").unwrap().is_none());
16730        }
16731    }
16732
16733    /// `scope.sync` and `scope.describe` through the real control handler: who
16734    /// may sync is decided by the registration and launch nonce of the module
16735    /// connection, never by the request body.
16736    mod scopes {
16737        mod idle_tests {
16738            use super::*;
16739            include!("scope_idle_tests.rs");
16740        }
16741        mod route_open_timings {
16742            use super::*;
16743            include!("route_open_timing_tests.rs");
16744        }
16745        mod route_open_refusal_log {
16746            use super::*;
16747            include!("route_open_refusal_log_tests.rs");
16748        }
16749        mod activity {
16750            include!("activity_scope_tests.rs");
16751        }
16752        mod delta {
16753            include!("scope_handler_delta_tests.rs");
16754        }
16755        mod call_modes {
16756            include!("scope_call_mode_tests.rs");
16757        }
16758        #[tokio::test]
16759        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16760            let rig = rig().await;
16761            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16762                "s".into(),
16763                1,
16764                "tool".into(),
16765            ))
16766            .unwrap();
16767            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16768            let (direct, _rx) = wide_ctx(999);
16769            for (ctx, expected) in [
16770                (&rig.owner, FrameType::Response),
16771                (&direct, FrameType::Error),
16772            ] {
16773                let frame =
16774                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16775                        .unwrap();
16776                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16777                assert_eq!(replies[0].header.ty, expected);
16778                if expected == FrameType::Response {
16779                    let reply: subc_control::ScopeServedByReply =
16780                        serde_json::from_slice(&replies[0].body).unwrap();
16781                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16782                } else {
16783                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16784                }
16785            }
16786            let reply = call(
16787                &rig.handler,
16788                &rig.owner,
16789                &ModuleControlRequestFromModule::ScopeDescribe {
16790                    owner: Principal::Reserved {
16791                        module_id: OWNER.into(),
16792                    },
16793                    scope_ref: "s".into(),
16794                },
16795            )
16796            .await;
16797            assert!(matches!(
16798                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16799                ModuleControlResponseToModule::ScopeDescribe { .. }
16800            ));
16801        }
16802
16803        use subc_protocol::scope::{
16804            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16805            ScopeStatus,
16806        };
16807
16808        use super::*;
16809
16810        const OWNER: &str = "prefrontal-core";
16811
16812        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16813            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16814        }
16815
16816        async fn call(
16817            handler: &ControlHandler,
16818            ctx: &RouteCtx,
16819            request: &ModuleControlRequestFromModule,
16820        ) -> Frame {
16821            let body = serde_json::to_vec(request).unwrap();
16822            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16823            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16824            assert_eq!(replies.len(), 1, "{replies:?}");
16825            replies.pop().unwrap()
16826        }
16827
16828        async fn sync(
16829            handler: &ControlHandler,
16830            ctx: &RouteCtx,
16831            generation: u64,
16832            scopes: Vec<ScopeRecord>,
16833        ) -> Result<ModuleControlResponseToModule, String> {
16834            let reply = call(
16835                handler,
16836                ctx,
16837                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16838            )
16839            .await;
16840            match reply.header.ty {
16841                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16842                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16843            }
16844        }
16845
16846        async fn describe(
16847            handler: &ControlHandler,
16848            ctx: &RouteCtx,
16849            owner: &str,
16850            scope_ref: &str,
16851        ) -> ModuleControlResponseToModule {
16852            let reply = call(
16853                handler,
16854                ctx,
16855                &ModuleControlRequestFromModule::ScopeDescribe {
16856                    owner: Principal::Reserved {
16857                        module_id: owner.to_string(),
16858                    },
16859                    scope_ref: scope_ref.to_string(),
16860                },
16861            )
16862            .await;
16863            assert_eq!(
16864                reply.header.ty,
16865                FrameType::Response,
16866                "{:?}",
16867                parse_error(&reply)
16868            );
16869            serde_json::from_slice(&reply.body).unwrap()
16870        }
16871
16872        #[tokio::test]
16873        async fn scope_describe_never_echoes_bind_proof_or_hash() {
16874            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16875            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16876            let mut rig = rig().await;
16877            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16878                .await;
16879            let _bound = rig
16880                .bound(
16881                    Some(AFT),
16882                    PLEXUS,
16883                    Some(
16884                        rig.selector("s", Some(1))
16885                            .with_bind_proof(Some(PROOF.into())),
16886                    ),
16887                )
16888                .await;
16889            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16890            let output = serde_json::to_string(&reply).unwrap();
16891            assert!(output.contains("\"status\":\"live\""));
16892            assert!(!output.contains(PROOF));
16893            assert!(!output.contains(HASH));
16894        }
16895
16896        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
16897        async fn module(
16898            handler: &ControlHandler,
16899            connection: u64,
16900            module_id: &str,
16901            nonce: Option<&str>,
16902        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16903            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16904            hello_via_sink(
16905                handler,
16906                &ctx,
16907                &mut rx,
16908                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16909            )
16910            .await;
16911            (ctx, rx)
16912        }
16913
16914        /// `direct` and every other client connection has no registration, so
16915        /// it can neither sync nor own a scope.
16916        #[tokio::test]
16917        async fn a_client_connection_cannot_sync_or_describe() {
16918            let handler = ControlHandler::new(Arc::new(Registry::default()));
16919            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16920            for request in [
16921                ModuleControlRequestFromModule::ScopeSync {
16922                    generation: 1,
16923                    scopes: vec![head("s", 1)],
16924                },
16925                ModuleControlRequestFromModule::ScopeDescribe {
16926                    owner: Principal::Direct,
16927                    scope_ref: "s".to_string(),
16928                },
16929            ] {
16930                let reply = call(&handler, &ctx, &request).await;
16931                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16932            }
16933            assert!(
16934                !handler
16935                    .scopes
16936                    .read()
16937                    .unwrap()
16938                    .describe(
16939                        &Principal::Reserved {
16940                            module_id: OWNER.to_string()
16941                        },
16942                        "s"
16943                    )
16944                    .owner_synced
16945            );
16946        }
16947
16948        /// A module the supervisor did not spawn registers without a launch
16949        /// nonce, so it is never an owner's current launch.
16950        #[tokio::test]
16951        async fn a_module_without_a_supervised_launch_cannot_sync() {
16952            let handler = ControlHandler::new(Arc::new(Registry::default()));
16953            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16954            assert_eq!(
16955                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16956                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16957            );
16958        }
16959
16960        #[tokio::test]
16961        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16962            let supervisor = SupervisorHandle::new();
16963            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16964            let handler = ControlHandler::new(Arc::new(Registry::default()))
16965                .with_supervisor(supervisor.clone());
16966            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16967            sync(&handler, &incumbent, 1, vec![head("s", 1)])
16968                .await
16969                .expect("the current launch syncs");
16970
16971            // A swap candidate registers with the swap token and is refused
16972            // while the incumbent keeps syncing.
16973            supervisor.open_swap(OWNER, "n2".to_string());
16974            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16975            assert_eq!(
16976                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16977                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16978            );
16979            sync(&handler, &incumbent, 2, vec![head("s", 1)])
16980                .await
16981                .expect("the serving owner syncs during the swap");
16982
16983            // The swap fails and is rolled back. The candidate never held sync
16984            // authority, and still cannot sync.
16985            supervisor.close_swap(OWNER);
16986            assert_eq!(
16987                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16988                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16989            );
16990            sync(&handler, &incumbent, 3, vec![head("s", 1)])
16991                .await
16992                .expect("the serving owner syncs after the rollback");
16993            handler.cleanup_connection(candidate.connection_id).unwrap();
16994
16995            // A swap that cuts over. Promotion records the candidate's nonce as
16996            // the module's spawn nonce, which is what `set_spawn_nonce` does
16997            // here; the promoted connection then takes authority at any
16998            // generation and the superseded incumbent is refused.
16999            supervisor.open_swap(OWNER, "n3".to_string());
17000            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17001            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17002            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17003                .await
17004                .expect("the promoted launch takes authority");
17005            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17006                panic!("unexpected reply {reply:?}");
17007            };
17008            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17009            assert_eq!(
17010                sync(&handler, &incumbent, 4, Vec::new()).await,
17011                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17012            );
17013        }
17014
17015        /// Authority dies with its connection: the cleanup path releases it,
17016        /// so the owner's next connection takes it at any generation.
17017        #[tokio::test]
17018        async fn closing_the_authority_connection_frees_sync_authority() {
17019            let supervisor = SupervisorHandle::new();
17020            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17021            let handler = ControlHandler::new(Arc::new(Registry::default()))
17022                .with_supervisor(supervisor.clone());
17023            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17024            sync(&handler, &first, 10, vec![head("s", 1)])
17025                .await
17026                .unwrap();
17027            handler.cleanup_connection(first.connection_id).unwrap();
17028
17029            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17030            sync(&handler, &second, 1, vec![head("s", 1)])
17031                .await
17032                .expect("the next connection takes the released authority");
17033        }
17034
17035        #[tokio::test]
17036        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17037            let supervisor = SupervisorHandle::new();
17038            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17039            let handler =
17040                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17041            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17042            sync(&handler, &first, 10, vec![head("s", 1)])
17043                .await
17044                .unwrap();
17045            handler
17046                .handle_control_frame(
17047                    &first,
17048                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17049                )
17050                .await
17051                .unwrap();
17052            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17053            sync(&handler, &second, 1, vec![head("s", 1)])
17054                .await
17055                .expect("GOODBYE releases authority even if the old socket remains open");
17056        }
17057
17058        #[tokio::test]
17059        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17060            let registry = Arc::new(Registry::default());
17061            let supervisor_handle = SupervisorHandle::new();
17062            let supervisor =
17063                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17064                    .with_handle(supervisor_handle.clone())
17065                    .with_daemon_incarnation("incarnation-7".to_string());
17066            // Configured with enabled: false, so the supervisor lists the
17067            // module without spawning a process for it.
17068            supervisor
17069                .supervise_configured(
17070                    ModuleSpec {
17071                        module_id: OWNER.to_string(),
17072                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17073                        args: Vec::new(),
17074                        env: Vec::new(),
17075                        reserved: false,
17076                        reserved_prefixes: Vec::new(),
17077                        protocol: ModuleProtocol::Subc,
17078                        overlap: Default::default(),
17079                    },
17080                    false,
17081                )
17082                .unwrap();
17083            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17084            let handler =
17085                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17086            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17087
17088            // Configured but not yet synced: a reader waits for the owner.
17089            let ModuleControlResponseToModule::ScopeDescribe {
17090                status,
17091                daemon_incarnation,
17092                owner_synced,
17093                owner_configured,
17094                scope,
17095                ..
17096            } = describe(&handler, &reader, OWNER, "s").await
17097            else {
17098                panic!("not a describe reply");
17099            };
17100            assert_eq!(status, ScopeStatus::NotLive);
17101            assert_eq!(daemon_incarnation, "incarnation-7");
17102            assert!(!owner_synced);
17103            assert!(owner_configured);
17104            assert!(scope.is_none());
17105
17106            // Not a supervised module: the owner will never sync, and a reader
17107            // refuses rather than waits.
17108            let ModuleControlResponseToModule::ScopeDescribe {
17109                status,
17110                owner_configured,
17111                ..
17112            } = describe(&handler, &reader, "ghost", "s").await
17113            else {
17114                panic!("not a describe reply");
17115            };
17116            assert_eq!(status, ScopeStatus::NotLive);
17117            assert!(!owner_configured);
17118
17119            // Live, with the stamp fields and the computed owner_authorized.
17120            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17121            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17122            let ModuleControlResponseToModule::ScopeDescribe {
17123                status,
17124                scope_epoch,
17125                owner_synced,
17126                scope,
17127                ..
17128            } = describe(&handler, &reader, OWNER, "s").await
17129            else {
17130                panic!("not a describe reply");
17131            };
17132            assert_eq!(status, ScopeStatus::Live);
17133            assert_eq!(scope_epoch, Some(4));
17134            assert!(owner_synced);
17135            let stamp = scope.expect("a live scope carries its stamp");
17136            assert!(
17137                stamp.owner_authorized,
17138                "prefrontal-core is the default authority"
17139            );
17140            assert_eq!(stamp.kind, ScopeKind::Head);
17141        }
17142
17143        #[tokio::test]
17144        async fn scope_authority_owners_decides_owner_authorized() {
17145            let supervisor = SupervisorHandle::new();
17146            supervisor.set_spawn_nonce("broca", "b1".to_string());
17147            let handler = ControlHandler::new(Arc::new(Registry::default()))
17148                .with_supervisor(supervisor)
17149                .with_scope_authority_owners(vec!["broca".to_string()]);
17150            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17151            let mut gated = head("s", 1);
17152            gated.attributes.agent_id = Some("agent".to_string());
17153            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17154            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17155                describe(&handler, &broca, "broca", "s").await
17156            else {
17157                panic!("not a describe reply");
17158            };
17159            assert!(scope.unwrap().owner_authorized);
17160        }
17161
17162        /// With route admission, the stamp, the commit re-check and drains in
17163        /// place, the feature is advertised: the module ops in HELLO_ACK, and
17164        /// `scopes/v1` in HELLO_ACK and `server.describe`.
17165        #[tokio::test]
17166        async fn scope_ops_and_the_scopes_capability_are_advertised() {
17167            let handler = ControlHandler::new(Arc::new(Registry::default()));
17168            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17169            let ack = hello_via_sink(
17170                &handler,
17171                &ctx,
17172                &mut rx,
17173                hello_frame("m", PROTOCOL_VERSION, 1),
17174            )
17175            .await;
17176            let ack = parse_ack(&ack);
17177            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17178                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17179            }
17180            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17181
17182            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17183            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17184            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17185            let reply = handler
17186                .handle_control_frame(&client, frame)
17187                .await
17188                .unwrap()
17189                .pop()
17190                .unwrap();
17191            let ClientControlResponse::ServerDescribe { capabilities, .. } =
17192                serde_json::from_slice(&reply.body).unwrap()
17193            else {
17194                panic!("not a server.describe reply");
17195            };
17196            assert!(
17197                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17198                "{capabilities:?}"
17199            );
17200        }
17201
17202        // ---- route admission, stamps, commit re-check and drains ----------
17203
17204        const PLEXUS: &str = "plexus";
17205        const OTHER: &str = "other";
17206        const AFT: &str = "aft";
17207        const BROCA: &str = "broca";
17208        const MAGIC: &str = "magic-context";
17209
17210        fn nonce(module_id: &str) -> String {
17211            format!("nonce-{module_id}")
17212        }
17213
17214        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17215            let (tx, rx) = mpsc::channel(64);
17216            (
17217                RouteCtx {
17218                    connection_id: ConnectionId::new(connection),
17219                    egress: FrameSink::new(tx),
17220                },
17221                rx,
17222            )
17223        }
17224
17225        /// A daemon with a configured owner (prefrontal-core) registered on its
17226        /// own module connection, two routable targets (plexus, other), and
17227        /// launch nonces minted for the modules that open routes as carriers.
17228        struct Rig {
17229            handler: ControlHandler,
17230            forwarding: Arc<ForwardingTable>,
17231            owner: RouteCtx,
17232            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17233            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17234            generation: u64,
17235            next_connection: u64,
17236            _supervisor: Supervisor,
17237        }
17238
17239        async fn rig() -> Rig {
17240            rig_with_flow_support(true).await
17241        }
17242
17243        async fn rig_with_flow_support(flow_support: bool) -> Rig {
17244            rig_with_scope_capabilities(flow_support, false).await
17245        }
17246
17247        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17248            let mut provides = Vec::new();
17249            if flow_support {
17250                provides.push("flow-scopes/v1");
17251            }
17252            if run_support {
17253                provides.push("agent-run-scopes/v1");
17254            }
17255            rig_with_target_capabilities(&provides).await
17256        }
17257
17258        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17259            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17260            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17261            // Decoder versions alone cannot admit scoped calls: only a capability
17262            // declaration promises the required run or flow behaviour.
17263            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17264            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17265            Frame::build(
17266                FrameType::Hello,
17267                control_flags(),
17268                0,
17269                0,
17270                connection,
17271                serde_json::to_vec(&body).unwrap(),
17272            )
17273            .unwrap()
17274        }
17275
17276        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17277            let registry = Arc::new(Registry::default());
17278            let forwarding = Arc::new(ForwardingTable::default());
17279            let supervisor_handle = SupervisorHandle::new();
17280            let supervisor =
17281                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17282                    .with_handle(supervisor_handle.clone());
17283            supervisor
17284                .supervise_configured(
17285                    ModuleSpec {
17286                        module_id: OWNER.to_string(),
17287                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17288                        args: Vec::new(),
17289                        env: Vec::new(),
17290                        reserved: false,
17291                        reserved_prefixes: Vec::new(),
17292                        protocol: ModuleProtocol::Subc,
17293                        overlap: Default::default(),
17294                    },
17295                    false,
17296                )
17297                .unwrap();
17298            for module_id in [OWNER, AFT, BROCA, MAGIC] {
17299                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17300            }
17301            let handler =
17302                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17303                    .with_supervisor(supervisor_handle);
17304            let (owner, mut owner_rx) = wide_ctx(1);
17305            hello_via_sink(
17306                &handler,
17307                &owner,
17308                &mut owner_rx,
17309                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17310            )
17311            .await;
17312            let mut modules = BTreeMap::new();
17313            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17314                let (ctx, mut rx) = wide_ctx(connection);
17315                let hello = hello_with_capabilities(module_id, connection, provides);
17316                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17317                modules.insert(module_id.to_string(), (ctx, rx));
17318            }
17319            Rig {
17320                handler,
17321                forwarding,
17322                owner,
17323                _owner_rx: owner_rx,
17324                modules,
17325                generation: 0,
17326                next_connection: 100,
17327                _supervisor: supervisor,
17328            }
17329        }
17330
17331        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17332            ScopeCarrier::new(Principal::Reserved {
17333                module_id: module_id.to_string(),
17334            })
17335            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17336        }
17337
17338        /// The scope most tests open under: aft carries to any module, broca
17339        /// only to plexus and other, and the owner delegates as agent-1.
17340        fn session(scope_epoch: u64) -> ScopeRecord {
17341            let mut record = head("s", scope_epoch);
17342            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17343            record.attributes.agent_id = Some("agent-1".to_string());
17344            record.attributes.delegates = true;
17345            record
17346        }
17347
17348        impl Rig {
17349            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17350                self.generation += 1;
17351                sync(&self.handler, &self.owner, self.generation, scopes)
17352                    .await
17353                    .expect("the owner's sync is accepted");
17354            }
17355
17356            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17357                ScopeSelector::new(
17358                    Principal::Reserved {
17359                        module_id: OWNER.to_string(),
17360                    },
17361                    scope_ref.to_string(),
17362                    scope_epoch,
17363                )
17364            }
17365
17366            fn open_frame(
17367                &mut self,
17368                opener: Option<&str>,
17369                target: &str,
17370                scope: Option<ScopeSelector>,
17371            ) -> (
17372                RouteCtx,
17373                mpsc::Receiver<crate::router::OutboundFrame>,
17374                Frame,
17375            ) {
17376                self.next_connection += 1;
17377                let (ctx, rx) = wide_ctx(self.next_connection);
17378                let root = unique_project_root("scoped-open");
17379                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17380                    target: RouteTarget::ToolProvider {
17381                        module_id: target.to_string(),
17382                    },
17383                    identity: BindIdentity::new(
17384                        root.path().to_path_buf(),
17385                        "unit".to_string(),
17386                        "session".to_string(),
17387                    ),
17388                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
17389                        module_id: module_id.to_string(),
17390                        launch_nonce: nonce(module_id),
17391                    }),
17392                    consumer_capabilities: None,
17393                    role_versions: None,
17394                    admission_facts: None,
17395                    scope,
17396                })
17397                .unwrap();
17398                let frame = Frame::build(
17399                    FrameType::Request,
17400                    control_flags(),
17401                    0,
17402                    0,
17403                    self.next_connection,
17404                    body,
17405                )
17406                .unwrap();
17407                (ctx, rx, frame)
17408            }
17409
17410            /// Open and expect a refusal before anything is relayed.
17411            async fn refused(
17412                &mut self,
17413                opener: Option<&str>,
17414                target: &str,
17415                scope: Option<ScopeSelector>,
17416            ) -> String {
17417                self.refusal_body(opener, target, scope).await["code"]
17418                    .as_str()
17419                    .unwrap()
17420                    .to_string()
17421            }
17422
17423            async fn refusal_body(
17424                &mut self,
17425                opener: Option<&str>,
17426                target: &str,
17427                scope: Option<ScopeSelector>,
17428            ) -> Value {
17429                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17430                let replies = tokio::time::timeout(
17431                    Duration::from_secs(2),
17432                    self.handler.handle_control_frame(&ctx, frame),
17433                )
17434                .await
17435                .expect("the open must be refused before waiting for a bind ack")
17436                .unwrap();
17437                assert_eq!(replies.len(), 1, "{replies:?}");
17438                assert_eq!(replies[0].header.ty, FrameType::Error);
17439                let (_, module_rx) = self.modules.get_mut(target).unwrap();
17440                assert!(
17441                    module_rx.try_recv().is_err(),
17442                    "a refused open relays nothing"
17443                );
17444                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17445                parse_error(&replies[0])
17446            }
17447
17448            /// Start an open and return its task and the bind the target got.
17449            async fn relayed(
17450                &mut self,
17451                opener: Option<&str>,
17452                target: &str,
17453                scope: Option<ScopeSelector>,
17454            ) -> Relayed {
17455                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17456                let handler = self.handler.clone();
17457                let task_ctx = ctx.clone();
17458                let task = tokio::spawn(async move {
17459                    handler
17460                        .handle_control_frame(&task_ctx, frame)
17461                        .await
17462                        .unwrap()
17463                });
17464                let (_, module_rx) = self.modules.get_mut(target).unwrap();
17465                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17466                    .await
17467                    .expect("the target receives the relayed route.bind")
17468                    .unwrap()
17469                    .frame;
17470                Relayed {
17471                    target: target.to_string(),
17472                    client: ctx,
17473                    client_rx: rx,
17474                    task,
17475                    bind,
17476                }
17477            }
17478
17479            async fn ack(&self, relayed: &Relayed) {
17480                let (module, _) = &self.modules[&relayed.target];
17481                self.handler
17482                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17483                    .await
17484                    .unwrap();
17485            }
17486
17487            /// Open, ack and return the bound route.
17488            async fn bound(
17489                &mut self,
17490                opener: Option<&str>,
17491                target: &str,
17492                scope: Option<ScopeSelector>,
17493            ) -> Bound {
17494                let relayed = self.relayed(opener, target, scope).await;
17495                self.ack(&relayed).await;
17496                let Relayed {
17497                    target,
17498                    client,
17499                    mut client_rx,
17500                    task,
17501                    bind,
17502                } = relayed;
17503                assert!(
17504                    task.await.unwrap().is_empty(),
17505                    "the open is answered by commit"
17506                );
17507                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17508                Bound {
17509                    target,
17510                    client,
17511                    client_rx,
17512                    channel,
17513                    epoch,
17514                    bind,
17515                }
17516            }
17517
17518            fn live(&self, route: &Bound) -> bool {
17519                matches!(
17520                    self.forwarding
17521                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17522                        .unwrap(),
17523                    DataRoute::Client(DataRouteState::Bound(_))
17524                )
17525            }
17526        }
17527
17528        struct Relayed {
17529            target: String,
17530            client: RouteCtx,
17531            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17532            task: tokio::task::JoinHandle<Vec<Frame>>,
17533            bind: Frame,
17534        }
17535
17536        struct Bound {
17537            target: String,
17538            client: RouteCtx,
17539            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17540            channel: u16,
17541            epoch: u32,
17542            bind: Frame,
17543        }
17544
17545        impl Bound {
17546            /// The reason of the `route.closed` this client was sent, after
17547            /// checking it also got a GOODBYE on exactly this route.
17548            fn closed_reason(&mut self) -> RouteCloseReason {
17549                let mut reason = None;
17550                let mut goodbye = false;
17551                while let Ok(outbound) = self.client_rx.try_recv() {
17552                    let frame = outbound.frame;
17553                    match frame.header.ty {
17554                        FrameType::Goodbye => {
17555                            assert_eq!(
17556                                (frame.header.channel, frame.header.epoch),
17557                                (self.channel, self.epoch)
17558                            );
17559                            goodbye = true;
17560                        }
17561                        FrameType::Push => {
17562                            let ClientControlPush::RouteClosed {
17563                                reason: r,
17564                                module_id,
17565                                ..
17566                            } = serde_json::from_slice(&frame.body).unwrap()
17567                            else {
17568                                panic!("unexpected push");
17569                            };
17570                            assert_eq!(module_id, self.target);
17571                            reason = Some(r);
17572                        }
17573                        other => panic!("unexpected frame {other:?}"),
17574                    }
17575                }
17576                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17577                reason.expect("the client is told why the route closed")
17578            }
17579
17580            fn untouched(&mut self) -> bool {
17581                self.client_rx.try_recv().is_err()
17582            }
17583
17584            fn stamp(&self) -> Option<ScopeStamp> {
17585                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17586                    ModuleControlRequest::RouteBind { scope, .. } => scope,
17587                    other => panic!("expected a route.bind, got {other:?}"),
17588                }
17589            }
17590        }
17591
17592        #[tokio::test]
17593        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17594        ) {
17595            let mut rig = rig().await;
17596            let mut record = session(1);
17597            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17598            record.child_owners = vec![Principal::Reserved {
17599                module_id: MAGIC.to_string(),
17600            }];
17601            rig.sync(vec![record]).await;
17602            let scope = || Some(rig_selector("s", Some(1)));
17603
17604            // Admitted: the owner, a bare carrier to any module, a targeted
17605            // carrier to its listed module.
17606            rig.bound(Some(OWNER), PLEXUS, scope()).await;
17607            rig.bound(Some(AFT), OTHER, scope()).await;
17608            rig.bound(Some(BROCA), PLEXUS, scope()).await;
17609
17610            // Refused scope_not_carrier: a targeted carrier to an unlisted
17611            // module, a module that is not listed at all (a child owner is not
17612            // a carrier), and a direct key-holder.
17613            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17614                assert_eq!(
17615                    rig.refused(opener, target, scope()).await,
17616                    error_codes::SCOPE_NOT_CARRIER,
17617                    "{opener:?} -> {target}"
17618                );
17619            }
17620        }
17621
17622        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17623            ScopeSelector::new(
17624                Principal::Reserved {
17625                    module_id: OWNER.to_string(),
17626                },
17627                scope_ref.to_string(),
17628                scope_epoch,
17629            )
17630        }
17631
17632        #[tokio::test]
17633        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17634            let mut rig = rig().await;
17635            rig.sync(vec![session(1)]).await;
17636            for opener in [OWNER, AFT] {
17637                assert_eq!(
17638                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17639                        .await,
17640                    error_codes::SCOPE_EPOCH_REQUIRED,
17641                    "{opener}"
17642                );
17643            }
17644        }
17645
17646        #[tokio::test]
17647        async fn admission_separates_not_synced_not_live_and_ended() {
17648            let mut rig = rig().await;
17649            // Before the configured owner's first sync: retryable.
17650            let code = rig
17651                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17652                .await;
17653            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17654            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17655
17656            // An owner that is not configured will never sync: terminal.
17657            let ghost = ScopeSelector::new(
17658                Principal::Reserved {
17659                    module_id: "ghost".to_string(),
17660                },
17661                "s".to_string(),
17662                Some(1),
17663            );
17664            assert_eq!(
17665                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17666                error_codes::SCOPE_NOT_LIVE
17667            );
17668
17669            rig.sync(vec![session(2)]).await;
17670            assert_eq!(
17671                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17672                    .await,
17673                error_codes::SCOPE_NOT_LIVE
17674            );
17675            for epoch in [1, 3] {
17676                assert_eq!(
17677                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17678                        .await,
17679                    error_codes::SCOPE_ENDED,
17680                    "epoch {epoch}"
17681                );
17682            }
17683            // Control: the live epoch is admitted.
17684            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17685                .await;
17686        }
17687
17688        #[tokio::test]
17689        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17690            let mut rig = rig().await;
17691            rig.sync(vec![session(1)]).await;
17692            let route = rig
17693                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17694                .await;
17695            let stamp = route.stamp().expect("a scoped bind carries the stamp");
17696            assert_eq!(stamp.scope_ref, "s");
17697            assert_eq!(stamp.scope_epoch, 1);
17698            assert_eq!(stamp.kind, ScopeKind::Head);
17699            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17700            assert!(stamp.attributes.delegates);
17701            assert!(stamp.owner_authorized);
17702            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17703            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17704
17705            // broca owns a scope of its own on its own module connection; it is
17706            // not in scope_authority_owners, so its stamp is not authorized.
17707            let (broca, mut broca_rx) = wide_ctx(50);
17708            hello_via_sink(
17709                &rig.handler,
17710                &broca,
17711                &mut broca_rx,
17712                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17713            )
17714            .await;
17715            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17716                .await
17717                .unwrap();
17718            let own = ScopeSelector::new(
17719                Principal::Reserved {
17720                    module_id: BROCA.to_string(),
17721                },
17722                "b".to_string(),
17723                Some(1),
17724            );
17725            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17726            assert!(!route.stamp().unwrap().owner_authorized);
17727        }
17728
17729        #[tokio::test]
17730        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17731            let mut rig = rig().await;
17732            let mut record = head("s", 1);
17733            record.carriers = vec![carrier(AFT, None)];
17734            let flow_id = "Flow:run-7/step_2!~";
17735            record.attributes.flow_id = Some(flow_id.to_string());
17736            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17737                .await
17738                .unwrap();
17739            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17740                panic!("not a sync reply");
17741            };
17742            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17743            let route = rig
17744                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17745                .await;
17746            assert!(rig.live(&route), "the stamped bind committed");
17747            let stamp = route.stamp().expect("a flow scope carries a stamp");
17748            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17749            assert_eq!(stamp.attributes.agent_id, None);
17750            assert!(!stamp.attributes.delegates);
17751            assert!(stamp.owner_authorized);
17752            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17753            assert_eq!(unscoped.stamp(), None);
17754        }
17755
17756        #[tokio::test]
17757        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17758            let mut rig = rig_with_flow_support(false).await;
17759            let mut record = session(1);
17760            record.attributes.flow_id = Some("flow:7".to_string());
17761            rig.sync(vec![record]).await;
17762            for opener in [OWNER, AFT] {
17763                let body = rig
17764                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17765                    .await;
17766                assert_eq!(body["code"], "target_flow_unsupported");
17767                let message = body["message"].as_str().unwrap();
17768                for required in [PLEXUS, "flow-scopes/v1"] {
17769                    assert!(message.contains(required), "{message}");
17770                }
17771            }
17772        }
17773
17774        #[tokio::test]
17775        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17776            let mut rig = rig_with_flow_support(true).await;
17777            let mut record = session(1);
17778            record.attributes.flow_id = Some("flow:7".to_string());
17779            rig.sync(vec![record]).await;
17780            for opener in [OWNER, AFT] {
17781                let route = rig
17782                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17783                    .await;
17784                assert!(rig.live(&route));
17785                assert_eq!(
17786                    route.stamp().unwrap().attributes.flow_id.as_deref(),
17787                    Some("flow:7")
17788                );
17789            }
17790        }
17791
17792        #[tokio::test]
17793        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17794            use std::future::Future;
17795
17796            let mut rig = rig_with_flow_support(true).await;
17797            let mut record = session(1);
17798            record.attributes.flow_id = Some("flow:7".to_string());
17799            rig.sync(vec![record]).await;
17800            let (client, mut client_rx, frame) =
17801                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17802            // Hold the route-open response permit so admission sees the first
17803            // target but relay reservation cannot capture an endpoint yet.
17804            for _ in 0..64 {
17805                client.egress.try_send(route_bind_ack(1)).unwrap();
17806            }
17807            let handler = rig.handler.clone();
17808            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17809            std::future::poll_fn(|cx| {
17810                assert!(open.as_mut().poll(cx).is_pending());
17811                std::task::Poll::Ready(())
17812            })
17813            .await;
17814            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17815
17816            let old_connection = rig.modules[PLEXUS].0.connection_id;
17817            rig.handler.cleanup_connection(old_connection).unwrap();
17818            let (replacement, mut replacement_rx) = wide_ctx(200);
17819            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17820            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17821            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17822            let hello = Frame::build(
17823                FrameType::Hello,
17824                control_flags(),
17825                0,
17826                0,
17827                200,
17828                serde_json::to_vec(&body).unwrap(),
17829            )
17830            .unwrap();
17831            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17832
17833            client_rx.try_recv().unwrap();
17834            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17835                .await
17836                .expect("the replacement is refused without waiting for a bind ack")
17837                .unwrap();
17838            assert_eq!(replies.len(), 1);
17839            let body = parse_error(&replies[0]);
17840            assert_eq!(body["code"], "target_flow_unsupported");
17841            for required in [PLEXUS, "flow-scopes/v1"] {
17842                assert!(body["message"].as_str().unwrap().contains(required));
17843            }
17844            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17845            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17846            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17847            assert!(rig
17848                .handler
17849                .registry
17850                .get_module_by_connection(replacement.connection_id)
17851                .unwrap()
17852                .is_some());
17853        }
17854
17855        #[tokio::test]
17856        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17857            use std::future::Future;
17858            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17859            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17860            const NEXT_HASH: &str =
17861                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17862            let mut rig = rig().await;
17863            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17864            record.carriers.push(ScopeCarrier::new(Principal::Direct));
17865            rig.sync(vec![record.clone()]).await;
17866            let (client, mut rx, frame) = rig.open_frame(
17867                None,
17868                PLEXUS,
17869                Some(
17870                    rig.selector("s", Some(1))
17871                        .with_bind_proof(Some(PROOF.into())),
17872                ),
17873            );
17874            // Hold egress reservation while the scope owner changes its bind-proof
17875            // hash, so a proof valid at admission must be checked again at relay.
17876            for _ in 0..64 {
17877                client.egress.try_send(route_bind_ack(1)).unwrap();
17878            }
17879            let handler = rig.handler.clone();
17880            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17881            std::future::poll_fn(|cx| {
17882                assert!(open.as_mut().poll(cx).is_pending());
17883                std::task::Poll::Ready(())
17884            })
17885            .await;
17886            record.bind_proof_sha256 = Some(NEXT_HASH.into());
17887            rig.sync(vec![record]).await;
17888            rx.try_recv().unwrap();
17889            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17890                .await
17891                .unwrap()
17892                .unwrap();
17893            assert_eq!(
17894                parse_error(&replies[0])["code"],
17895                error_codes::SCOPE_BIND_PROOF_MISMATCH
17896            );
17897            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17898            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17899            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17900        }
17901
17902        #[tokio::test]
17903        async fn scope_bind_proof_reserved_openers_ignore_proof() {
17904            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17905            let mut rig = rig().await;
17906            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17907                .await;
17908            for opener in [OWNER, AFT] {
17909                for proof in [None, Some("malformed+proof=")] {
17910                    let route = rig
17911                        .bound(
17912                            Some(opener),
17913                            PLEXUS,
17914                            Some(
17915                                rig.selector("s", Some(1))
17916                                    .with_bind_proof(proof.map(str::to_string)),
17917                            ),
17918                        )
17919                        .await;
17920                    assert!(rig.live(&route));
17921                }
17922            }
17923        }
17924
17925        #[tokio::test]
17926        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17927            use std::future::Future;
17928            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17929            const NEXT_HASH: &str =
17930                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17931            let mut rig = rig().await;
17932            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17933                .await;
17934            let (client, mut rx, frame) =
17935                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17936            for _ in 0..64 {
17937                client.egress.try_send(route_bind_ack(1)).unwrap();
17938            }
17939            let handler = rig.handler.clone();
17940            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17941            std::future::poll_fn(|cx| {
17942                assert!(open.as_mut().poll(cx).is_pending());
17943                std::task::Poll::Ready(())
17944            })
17945            .await;
17946            rig.sync(vec![
17947                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17948            ])
17949            .await;
17950            rx.try_recv().unwrap();
17951            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17952            let bind = std::future::poll_fn(|cx| {
17953                assert!(open.as_mut().poll(cx).is_pending());
17954                module_rx.poll_recv(cx)
17955            })
17956            .await
17957            .unwrap()
17958            .frame;
17959            handler
17960                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17961                .await
17962                .unwrap();
17963            let replies = tokio::time::timeout(Duration::from_secs(2), open)
17964                .await
17965                .unwrap()
17966                .unwrap();
17967            assert_eq!(replies.len(), 1);
17968            // Proof rechecking allowed the relay. Independent tag settlement
17969            // still asks the opener to retry after any same-epoch content change.
17970            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17971            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17972            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17973            let route = rig
17974                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17975                .await;
17976            assert!(rig.live(&route));
17977        }
17978
17979        #[tokio::test]
17980        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17981        {
17982            let mut rig = rig_with_flow_support(false).await;
17983            rig.sync(vec![session(1)]).await;
17984            let route = rig
17985                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17986                .await;
17987            assert!(rig.live(&route));
17988            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17989            let mut record = session(1);
17990            record.attributes.flow_id = Some("flow:7".to_string());
17991            rig.sync(vec![record]).await;
17992            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17993            assert!(rig.live(&unscoped));
17994            assert_eq!(unscoped.stamp(), None);
17995        }
17996
17997        #[tokio::test]
17998        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17999            let mut rig = rig().await;
18000            let mut record = session(1);
18001            record.attributes.flow_id = Some("flow:7".to_string());
18002            rig.sync(vec![record.clone()]).await;
18003            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18004            let mut owner_route = rig
18005                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18006                .await;
18007            let mut carrier_route = rig
18008                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18009                .await;
18010            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18011            rig.sync(vec![record.clone()]).await;
18012            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18013            assert!(rig.live(&owner_route) && owner_route.untouched());
18014            assert!(rig.live(&carrier_route) && carrier_route.untouched());
18015
18016            record.attributes.flow_id = Some("flow:8".to_string());
18017            rig.sync(vec![record]).await;
18018            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18019            let before = before.unwrap();
18020            assert_eq!(after.scope_epoch, before.scope_epoch);
18021            assert!(after.version > before.version);
18022            for route in [&mut owner_route, &mut carrier_route] {
18023                assert!(!rig.live(route));
18024                assert_eq!(
18025                    route.closed_reason(),
18026                    RouteCloseReason::ScopeDelegationChanged
18027                );
18028            }
18029            assert!(rig.live(&unscoped) && unscoped.untouched());
18030            // Each provider also receives a GOODBYE for its drained route;
18031            // consume it before expecting the next route.bind on that sink.
18032            for target in [PLEXUS, OTHER] {
18033                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18034                let goodbye = module_rx
18035                    .try_recv()
18036                    .expect("the provider sees the drain")
18037                    .frame;
18038                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18039                assert!(module_rx.try_recv().is_err());
18040            }
18041            let rebound = rig
18042                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18043                .await;
18044            assert_eq!(
18045                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18046                Some("flow:8")
18047            );
18048        }
18049
18050        /// The owner's sync lands between admission and the module's ack. The
18051        /// open is refused by name, the module's other routes stay up, and the
18052        /// reserved pair is released. Changed content is retryable; an ended
18053        /// scope is not.
18054        #[tokio::test]
18055        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18056            let mut rig = rig().await;
18057            rig.sync(vec![session(1)]).await;
18058            let mut cotenant = rig
18059                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18060                .await;
18061
18062            let mut changed = session(1);
18063            changed.child_owners.push(Principal::Reserved {
18064                module_id: MAGIC.to_string(),
18065            });
18066            let mut ended = None;
18067            for (code, next) in [
18068                (error_codes::SCOPE_CHANGED, vec![changed]),
18069                (error_codes::SCOPE_ENDED, Vec::new()),
18070            ] {
18071                let relayed = rig
18072                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18073                    .await;
18074                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18075                ended = Some(next.is_empty());
18076                rig.sync(next).await;
18077                rig.ack(&relayed).await;
18078                let replies = relayed.task.await.unwrap();
18079                assert_eq!(replies.len(), 1, "{replies:?}");
18080                assert_eq!(parse_error(&replies[0])["code"], code);
18081                assert_eq!(
18082                    subc_protocol::error_codes::is_retryable_route_open(code),
18083                    code == error_codes::SCOPE_CHANGED
18084                );
18085                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18086                // The module is told to drop just the binding it created.
18087                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18088                // Collected, because ending the scope also closes the co-tenant
18089                // route, whose GOODBYE comes first.
18090                let mut goodbyes = Vec::new();
18091                while let Ok(outbound) = plexus_rx.try_recv() {
18092                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18093                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18094                }
18095                assert!(
18096                    goodbyes.contains(&(bind_channel, bind_epoch)),
18097                    "{goodbyes:?}"
18098                );
18099                assert!(rig
18100                    .handler
18101                    .registry
18102                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18103                    .unwrap()
18104                    .is_some());
18105            }
18106            assert_eq!(ended, Some(true));
18107            // The co-tenant stayed up through the change, and closed only when
18108            // the scope ended, by the drain rule rather than by the commit.
18109            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18110        }
18111
18112        /// Each row of the drain table on one set of routes: the owner's, a
18113        /// bare carrier's, and a targeted carrier's to each of its targets.
18114        #[tokio::test]
18115        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18116            struct Case {
18117                name: &'static str,
18118                change: fn(&mut ScopeRecord),
18119                /// Closed routes by index: owner->plexus, aft->plexus,
18120                /// broca->plexus, broca->other.
18121                closed: [Option<RouteCloseReason>; 4],
18122            }
18123            use RouteCloseReason::*;
18124            let cases = [
18125                Case {
18126                    name: "a carrier entry removed",
18127                    change: |r| {
18128                        r.carriers.retain(|c| {
18129                            c.principal
18130                                != Principal::Reserved {
18131                                    module_id: AFT.to_string(),
18132                                }
18133                        })
18134                    },
18135                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18136                },
18137                Case {
18138                    name: "a target removed from a carrier",
18139                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18140                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
18141                },
18142                Case {
18143                    name: "a bare carrier narrowed to targets",
18144                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18145                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18146                },
18147                Case {
18148                    name: "delegates turned off",
18149                    change: |r| r.attributes.delegates = false,
18150                    closed: [Some(ScopeDelegationChanged); 4],
18151                },
18152                Case {
18153                    name: "agent_id changed",
18154                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18155                    closed: [Some(ScopeDelegationChanged); 4],
18156                },
18157                Case {
18158                    name: "a carrier added, child owners changed, the record re-sent",
18159                    change: |r| {
18160                        r.carriers.push(carrier(MAGIC, None));
18161                        r.child_owners.push(Principal::Reserved {
18162                            module_id: MAGIC.to_string(),
18163                        });
18164                    },
18165                    closed: [None; 4],
18166                },
18167                Case {
18168                    name: "a target added",
18169                    change: |r| {
18170                        r.carriers[1]
18171                            .targets
18172                            .as_mut()
18173                            .unwrap()
18174                            .push("third".to_string())
18175                    },
18176                    closed: [None; 4],
18177                },
18178                Case {
18179                    name: "delegates turned on",
18180                    change: |r| r.attributes.delegates = true,
18181                    closed: [None; 4],
18182                },
18183            ];
18184            for case in cases {
18185                let mut rig = rig().await;
18186                rig.sync(vec![session(1)]).await;
18187                let scope = || Some(rig_selector("s", Some(1)));
18188                let mut routes = [
18189                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
18190                    rig.bound(Some(AFT), PLEXUS, scope()).await,
18191                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
18192                    rig.bound(Some(BROCA), OTHER, scope()).await,
18193                ];
18194                let mut record = session(1);
18195                (case.change)(&mut record);
18196                rig.sync(vec![record]).await;
18197                for (index, expected) in case.closed.iter().enumerate() {
18198                    let route = &mut routes[index];
18199                    match expected {
18200                        Some(reason) => {
18201                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
18202                            assert_eq!(
18203                                route.closed_reason(),
18204                                *reason,
18205                                "{}: route {index}",
18206                                case.name
18207                            );
18208                        }
18209                        None => {
18210                            assert!(rig.live(route), "{}: route {index} closed", case.name);
18211                            assert!(
18212                                route.untouched(),
18213                                "{}: route {index} was told something",
18214                                case.name
18215                            );
18216                        }
18217                    }
18218                }
18219            }
18220        }
18221
18222        #[tokio::test]
18223        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18224            // Removed, and replaced by a higher epoch.
18225            for next in [Vec::new(), vec![session(2)]] {
18226                let mut rig = rig().await;
18227                rig.sync(vec![session(1)]).await;
18228                let mut route = rig
18229                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18230                    .await;
18231                rig.sync(next).await;
18232                assert!(!rig.live(&route));
18233                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18234            }
18235
18236            // A child whose parent ends: its routes close as parent-ended, the
18237            // child stays live, and routes under the parent close as ended.
18238            let mut rig = rig().await;
18239            let mut child = session(1);
18240            child.scope_ref = "child".to_string();
18241            child.kind = ScopeKind::Worker;
18242            child.parent = Some(ScopeParent::new(
18243                Principal::Reserved {
18244                    module_id: OWNER.to_string(),
18245                },
18246                "s".to_string(),
18247                1,
18248            ));
18249            rig.sync(vec![session(1), child.clone()]).await;
18250            let mut child_route = rig
18251                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18252                .await;
18253            assert_eq!(
18254                child_route.stamp().unwrap().parent_state,
18255                Some(ParentState::Linked)
18256            );
18257            rig.sync(vec![child]).await;
18258            assert!(!rig.live(&child_route));
18259            assert_eq!(
18260                child_route.closed_reason(),
18261                RouteCloseReason::ScopeParentEnded
18262            );
18263        }
18264
18265        #[tokio::test]
18266        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18267        ) {
18268            let mut rig = rig().await;
18269            rig.sync(vec![session(1)]).await;
18270            let mut route = rig
18271                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18272                .await;
18273            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18274            rig.sync(vec![session(1)]).await;
18275            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18276            assert!(rig.live(&route) && route.untouched());
18277
18278            // A call in flight on the route when another carrier is added. A
18279            // forwarded REQUEST holds one credit on the route's flow until the
18280            // module answers; the router takes it exactly like this.
18281            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18282                .forwarding
18283                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18284                .unwrap()
18285            else {
18286                panic!("the route is bound");
18287            };
18288            binding.flow.acquire_tagged(9, false).await.unwrap();
18289            let mut widened = session(1);
18290            widened.carriers.push(carrier(MAGIC, None));
18291            rig.sync(vec![widened]).await;
18292            assert!(rig.live(&route) && route.untouched());
18293            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18294            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18295            // The call's credit is still held on an open flow, so its answer
18296            // will be delivered: closing the route would have closed the flow.
18297            assert_eq!(binding.flow.in_flight(), 1);
18298            binding
18299                .flow
18300                .acquire_tagged(10, false)
18301                .await
18302                .expect("the flow is still open");
18303        }
18304
18305        /// A swap's superseded endpoint keeps its routes until drained; ending
18306        /// the scope closes them there too.
18307        #[tokio::test]
18308        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18309            let mut rig = rig().await;
18310            rig.sync(vec![session(1)]).await;
18311            let mut on_incumbent = rig
18312                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18313                .await;
18314
18315            // Swap plexus: register a candidate and cut over, leaving the
18316            // incumbent superseded with the route still on it.
18317            let (candidate, _candidate_rx) = wide_ctx(9);
18318            let registration = rig
18319                .handler
18320                .registry
18321                .register_candidate_with_control_ops(
18322                    manifest(PLEXUS, PROTOCOL_VERSION),
18323                    PROTOCOL_VERSION,
18324                    candidate.connection_id,
18325                    module_baseline_control_ops(),
18326                )
18327                .unwrap();
18328            rig.forwarding
18329                .register_candidate_module_connection(
18330                    candidate.connection_id,
18331                    PLEXUS.to_string(),
18332                    PROTOCOL_VERSION,
18333                    manifest_concurrency(&registration.manifest),
18334                    candidate.egress.clone(),
18335                )
18336                .unwrap();
18337            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18338            rig.handler
18339                .registry
18340                .promote_candidate(PLEXUS)
18341                .unwrap()
18342                .unwrap();
18343            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18344
18345            rig.sync(Vec::new()).await;
18346            assert!(!rig.live(&on_incumbent));
18347            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18348            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18349            let goodbye = incumbent_rx
18350                .try_recv()
18351                .expect("the superseded endpoint is told")
18352                .frame;
18353            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18354        }
18355    }
18356}
18357
18358#[cfg(test)]
18359mod concurrency_default_exposure_tests {
18360    use super::*;
18361
18362    fn hello_body(role_json: &str) -> Vec<u8> {
18363        format!(
18364            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18365        )
18366        .into_bytes()
18367    }
18368
18369    fn manifest_from(body: &[u8]) -> ModuleManifest {
18370        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18371        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18372            .expect("manifest parses")
18373    }
18374
18375    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18376
18377    #[test]
18378    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18379        let body = hello_body(&format!(
18380            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18381        ));
18382        let manifest = manifest_from(&body);
18383        // Precondition: serde really resolved it to the default, so the typed
18384        // manifest alone cannot answer the question this probe exists for.
18385        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18386        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18387    }
18388
18389    #[test]
18390    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18391        let body = hello_body(&format!(
18392            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18393        ));
18394        let manifest = manifest_from(&body);
18395        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18396        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18397    }
18398
18399    #[test]
18400    fn non_management_roles_are_never_reported() {
18401        let body = hello_body(
18402            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18403        );
18404        let manifest = manifest_from(&body);
18405        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18406    }
18407}