Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_confirm_operations,
25        validate_hello_event_declarations, validate_hello_flow_operations,
26        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29    },
30    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
31    scope::{
32        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
33        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
34        SCOPE_SYNC_OP,
35    },
36    session::{
37        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
38        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
39        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
40        ROLE_VERSIONS_FIELD,
41    },
42    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
43    Priority, RouteTarget, PROTOCOL_VERSION,
44};
45use tokio::time::{timeout_at, Instant};
46use tracing::{debug, info, instrument::WithSubscriber, warn};
47
48use crate::{
49    caller_identity::CallerIdentity,
50    capability_requirements::{
51        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
52        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
53        RequirementStatus, RuntimeModule,
54    },
55    child_roster::DaemonShutdownFlag,
56    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
57    forwarding::{
58        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
59        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
60        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
61    },
62    observability::{
63        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
64    },
65    privileged_prompt::{prompt_text, PrivilegedOperation},
66    provenance::{
67        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
68    },
69    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
70    router::{daemon_error_flags, RouteCtx, RouterError},
71    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
72    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
73    stderr_tail::{CaptureState, TailEntry},
74    supervise::{
75        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
76        SupervisorHandle, SwapHelloAdmission,
77    },
78    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
79};
80
81/// Lowest envelope version this subc build will negotiate.
82///
83/// Module HELLO negotiation is exact: peers must use the daemon's locked
84/// protocol version. Older and newer peers receive `version_unsupported` and
85/// are not registered.
86pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
87
88const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
89const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
90const CAP_PING_PONG: &str = "ping_pong_v1";
91const CAP_SESSION_ATTACH: &str = "session_attach_v1";
92const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
93
94const SUBC_CONTROL_OPS: &[&str] = &[
95    ops::DAEMON_SHUTDOWN_INTENT,
96    ops::ACTIVITY_RECORD,
97    ops::ACTIVITY_QUERY,
98    ops::ACTIVITY_SUBSCRIBE,
99    ops::SERVER_DESCRIBE,
100    ops::CATALOG_LIST,
101    ops::ROUTE_OPEN,
102    ops::ROUTE_POLL,
103    ops::ROUTE_CLOSING,
104    ops::ROUTE_CLOSED,
105    ops::SUPERVISOR_LIST,
106    ops::SUPERVISOR_RESTART,
107    ops::SUPERVISOR_SWAP,
108    ops::SUPERVISOR_RELOAD,
109    ops::SUPERVISOR_RESCAN,
110    ops::SUPERVISOR_RELEASE_RESERVED,
111    ops::SUPERVISOR_SET_ENABLED,
112    ops::SUPERVISOR_HEALTH_PROBE,
113    ops::SUPERVISOR_HEALTH,
114    ops::SUPERVISOR_STDERR_TAIL,
115    ops::SUPERVISOR_TERMINALS,
116    ops::SUPERVISOR_ROUTES,
117    ops::SUPERVISOR_PROVENANCE,
118    ops::SUPERVISOR_SPAWN_SNAPSHOT,
119    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
120];
121
122/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
123/// announced by a capability in `server.describe` instead, and clients check
124/// that capability before sending it. `ck daemon --json` prints `subc_ops`
125/// verbatim and scripts read that output, so a new name there would change it.
126/// A malformed body for one of these is still refused `invalid_control_body`
127/// rather than as an unknown op.
128const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
129
130mod rescan_checked;
131use rescan_checked::{RescanMode, RescanPlan};
132
133const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
134    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
135    "supervisor.live_roots",
136    SCOPE_SYNC_OP,
137    SCOPE_APPLY_OP,
138    SCOPE_DESCRIBE_OP,
139    "operator.confirm",
140];
141
142/// Module-originated ops the daemon answers but does not advertise in
143/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
144/// belongs to is incomplete, so no module is told it works before it does.
145const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
146
147const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
148
149/// How long subc waits for a module to ack a relayed route.bind before returning
150/// `module_timeout`. The ack waits on the module's own configure, which for AFT
151/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
152/// and DB-open work — on a cold page cache or a large repo that legitimately
153/// exceeds a couple of seconds. The default is generous because rejecting a VALID
154/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
155/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
156pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
157
158/// How many CONSECUTIVE full-budget relay timeouts against one target module
159/// open that module's bind-relay breaker.
160///
161/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
162/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
163/// so three consecutive full-budget timeouts take ~36s to observe: every client
164/// whose open contributed to opening the breaker had already given up on its
165/// own. That is what makes opening the breaker unable to turn a call that would
166/// have succeeded into a refusal — it can only make an already-failing module
167/// fail faster.
168///
169/// Two would be reachable inside one default deadline. One would convict a
170/// module on a single cold-cache bind, which is exactly the valid-but-slow case
171/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
172pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
173
174/// How long a module's bind-relay breaker stays open before exactly one
175/// `route.open` is let through as a probe.
176///
177/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
178/// re-pays a full-budget stall almost continuously, and the breaker stops being
179/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
180/// deadline: a client that starts retrying after the module recovers has to get
181/// a probe opportunity inside its own deadline, or the breaker converts a
182/// recovered module into a failed call — the failure it exists to prevent,
183/// pointed the other way.
184///
185/// 20s sits between those with room on both sides, and it caps what a wedged
186/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
187/// rather than one per `route.open` per connection. The stall that motivated
188/// this, with its measurements, is written up in
189/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
190/// waited the whole budget out.
191pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
192
193const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
194const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
195
196fn reload_verdict(
197    configured: &Path,
198    spawned_from: Option<&Path>,
199    image: subc_control::RunningImageAgreement,
200) -> PendingReloadVerdict {
201    let path = match spawned_from {
202        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
203            ReloadPathAgreement::Match
204        }
205        Some(spawned_from) => ReloadPathAgreement::Mismatch {
206            configured: configured.to_path_buf(),
207            spawned_from: spawned_from.to_path_buf(),
208        },
209        None => ReloadPathAgreement::Unavailable {
210            reason: if matches!(
211                image,
212                subc_control::RunningImageAgreement::Unavailable {
213                    reason: subc_control::RunningImageUnavailableReason::NotRunning
214                }
215            ) {
216                ReloadPathUnavailableReason::NotRunning
217            } else {
218                ReloadPathUnavailableReason::SpawnedPathUnavailable
219            },
220        },
221    };
222    PendingReloadVerdict {
223        path,
224        image,
225        spec: None,
226    }
227}
228
229fn launch_spec_agreement(
230    configured: &crate::supervise::ModuleSpec,
231    running: bool,
232    spawned: Option<&crate::supervise::LaunchSpecDigest>,
233) -> subc_control::LaunchSpecAgreement {
234    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
235    if !running {
236        return Agreement::Unavailable {
237            reason: Reason::NotRunning,
238        };
239    }
240    match spawned {
241        None => Agreement::Unavailable {
242            reason: Reason::SpawnedSpecUnavailable,
243        },
244        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
245            Agreement::Match
246        }
247        Some(_) => Agreement::Mismatch,
248    }
249}
250
251fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
252    if configured == spawned_from {
253        return true;
254    }
255    #[cfg(windows)]
256    {
257        // Windows spawn records the resolved verbatim path (\\?\...), while
258        // configuration keeps its original spelling. Resolve both with the OS
259        // instead of stripping prefixes or folding case, which could conflate
260        // distinct names in case-sensitive directories. Different resolved
261        // paths still differ even if their files have identical content.
262        match (
263            std::fs::canonicalize(configured),
264            std::fs::canonicalize(spawned_from),
265        ) {
266            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
267            _ => false,
268        }
269    }
270    #[cfg(not(windows))]
271    {
272        false
273    }
274}
275
276#[derive(Clone)]
277struct DaemonProvenanceFacts {
278    build: DaemonBuildProvenance,
279    pid: Option<u32>,
280    started_at_ms: Option<u64>,
281    start_clock: Option<crate::clock::StartClock>,
282    executable_path: Option<PathBuf>,
283    executable_identity: Option<SpawnedFileIdentity>,
284    process_start_time: Option<u64>,
285    probe: ExecutableIdentityProbe,
286}
287
288impl Default for DaemonProvenanceFacts {
289    fn default() -> Self {
290        Self {
291            build: DaemonBuildProvenance {
292                build_git_sha: None,
293                build_lock_digest: None,
294            },
295            pid: None,
296            started_at_ms: None,
297            start_clock: None,
298            executable_path: None,
299            executable_identity: None,
300            process_start_time: None,
301            probe: ExecutableIdentityProbe::default(),
302        }
303    }
304}
305
306#[derive(Debug, Clone)]
307struct SupervisorRescanContext {
308    supervisor: Supervisor,
309    config_path: PathBuf,
310    configured_port: Option<u16>,
311    storage_config: Option<crate::daemon_config::StorageConfig>,
312    admission_facts_carrier_module_id: Option<String>,
313    admission_facts_targets: Option<Vec<String>>,
314    scope_authority_owners: Vec<String>,
315    privileged_operator_prompt: PrivilegedOperatorPrompt,
316    /// Counts executed rescans that set out to change the supervised modules'
317    /// staged or applied configuration, counted before the first change so a
318    /// rescan that fails partway still counts. With the daemon incarnation it
319    /// names the state a `supervisor.rescan_checked` preview was computed
320    /// against.
321    generation: Arc<std::sync::atomic::AtomicU64>,
322}
323
324/// Refusal labels passed to `observe_route_open_refusal` that mean the target
325/// module is not serving right now, and so open or extend an outage in the
326/// route outage tracker. Every one of them is only reachable after the target
327/// was found in the registry, which is what keeps an arbitrary client-chosen
328/// id from ever creating tracker state.
329///
330/// Deliberately absent: `not_registered` and `removed` (the id may be
331/// anything a client sent, and a removed module is gone on purpose),
332/// `protocol_none` (such a module never serves routes, so nothing is out),
333/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
334/// capability and admission-facts refusals (they refuse the caller, not a
335/// module outage), and `relay_reservation_failed` (its code ranges over
336/// capacity limits as well as a vanished connection). Capacity, breaker,
337/// relay-timeout and module-rejection refusals do not pass through that
338/// function at all; the breaker logs its own transitions.
339///
340/// The two not-serving refusals that bypass that function record themselves
341/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
342/// `required_capability_unprovided` is not tracked: the module itself is up,
343/// and the outage belongs to the missing provider.
344const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
345    "reloading",
346    "supervisor_not_live",
347    "registration_not_active",
348    "no_forwarding_connection",
349    "relay_send_failed",
350];
351
352/// Real channel-0 control handler for subc itself.
353#[derive(Clone)]
354pub struct ControlHandler {
355    registry: Arc<Registry>,
356    activity: Option<crate::activity::ActivityLedger>,
357    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
358    forwarding: Arc<ForwardingTable>,
359    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
360    supervisor: SupervisorHandle,
361    subc_capabilities: Arc<[String]>,
362    /// Daemon-wide route.bind relay budget. Used as the fallback when the
363    /// target module has no per-module override in
364    /// `route_bind_relay_timeouts`.
365    route_bind_relay_timeout: Duration,
366    /// Per-module route.bind relay budget overrides, keyed by module id. When
367    /// `handle_route_open` resolves the deadline for a target module, a
368    /// per-module entry wins over the daemon-wide value above.
369    route_bind_relay_timeouts: BTreeMap<String, Duration>,
370    /// Per-target-module bind-relay breaker state. Shared with the forwarding
371    /// table, which is where a new module connection resets it.
372    route_bind_breakers: RouteBindBreakers,
373    /// Live relay admissions keyed by target module. Shared through the
374    /// forwarding table so cloned or separately built handlers enforce one cap.
375    route_bind_concurrency: RouteBindConcurrency,
376    /// Start and end of each module's not-serving period as seen by
377    /// `route.open`, so an outage gets one line at each edge instead of only
378    /// the per-refusal INFO lines. Taken from the forwarding table, so every
379    /// handler built over one table shares it.
380    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
381    /// Consecutive relay timeouts that open a module's breaker.
382    route_bind_breaker_threshold: u32,
383    /// How long a breaker stays open before one probe is admitted.
384    route_bind_breaker_cooldown: Duration,
385    health_probe_timeout: Duration,
386    /// Central storage policy. When set, each registering module receives its
387    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
388    storage_config: Option<crate::daemon_config::StorageConfig>,
389    /// The machine id established at boot, served on every HELLO_ACK and on
390    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
391    /// changes the file, never this value. `None` serves no id.
392    machine_id: Option<crate::machine_id::MachineId>,
393    admission_facts_carrier_module_id: Option<String>,
394    admission_facts_targets: Option<Vec<String>>,
395    /// Scope records with their sync authorities and tombstones; see
396    /// `crate::scopes`. Shared by clones of this handler, so every connection
397    /// reads and writes one table.
398    scopes: Arc<RwLock<ScopeTable>>,
399    /// Wakes the expiry loop when an accepted scope change may have added a
400    /// deadline, so the loop can wait without ticking while no live scope has
401    /// one. A permit is stored when nothing waits, so no wake is lost.
402    scope_deadline_added: Arc<tokio::sync::Notify>,
403    /// The configured `scope_authority_owners`, kept so a rescan can report a
404    /// changed value as needing a daemon restart; rescan never applies it.
405    scope_authority_owners: Vec<String>,
406    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
407    privileged_operator_prompt: PrivilegedOperatorPrompt,
408    /// The local and peer address of each accepted TCP client connection, read
409    /// from the daemon's own socket, so a privileged request can name its caller.
410    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
411    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
412    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
413    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
414    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
415    pub(crate) operator_apps:
416        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417    /// One launch entry per app, behind a leaf lock: no other lock is taken
418    /// while it is held. Entries are memory-only, so a daemon or
419    /// installed-service restart drops them and a relaunched daemon never
420    /// mistakes an older process for its own launch.
421    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
422    /// Serializes `app.open` launches: the liveness check in the decision and
423    /// the later publish must not interleave across concurrent calls, or two
424    /// calls could both launch and the second publish would orphan the first
425    /// process. Refusals and `already_running` answers share the same lock;
426    /// `app.open` is rare, so the cost is negligible.
427    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
428    // Portable wire samples exercise dispatch without starting an OS process.
429    #[cfg(test)]
430    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
431    #[cfg(test)]
432    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
433    /// Test-only Windows signature and flash hooks. Production builds carry
434    /// neither: with the publisher unset, the production table never reaches
435    /// verification, and the default attention hook only logs.
436    #[allow(dead_code)]
437    #[cfg(windows)]
438    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
439    #[allow(dead_code)]
440    #[cfg(windows)]
441    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
442    /// Test-only tracking fault hook: makes opening the tracking handle on
443    /// the launched pid fail, so a test can prove an untrackable launch is
444    /// stopped rather than left running.
445    #[allow(dead_code)]
446    #[cfg(windows)]
447    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
448    /// Test-only count of Windows spawn attempts, so a test can prove a
449    /// refused launch started no process. Unit tests only.
450    #[cfg(all(test, windows))]
451    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
452    #[cfg(test)]
453    caller_lookup: Option<TestCallerLookup>,
454    /// The launch nonce each module connection presented at HELLO, which is how
455    /// a `scope.sync` is matched to the owner's current launch.
456    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
457    rescan: Option<SupervisorRescanContext>,
458    connected_clients: ConnectedClients,
459    counters: DaemonCounters,
460    /// Why this daemon serves no local endpoint, when binding one failed. `None`
461    /// when the endpoint is served, and also on a handler that was never told.
462    local_endpoint_unavailable: Option<String>,
463    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
464    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
465    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
466    /// their usual severity (see [`log_requirement_events`]). A handler built
467    /// without one has a flag that is never set.
468    daemon_shutdown: DaemonShutdownFlag,
469    daemon_provenance: DaemonProvenanceFacts,
470    #[cfg(test)]
471    control_dispatch_delay: Option<Duration>,
472    #[cfg(test)]
473    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
474    /// Holds `route.open` after the route.bind is queued to the module and
475    /// before it is marked enqueued, so a test can make the module's reply (or
476    /// a client teardown) land exactly inside that window.
477    #[cfg(test)]
478    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
479    /// Holds a HELLO after its registry entry is reserved and before its
480    /// forwarding endpoint is installed, so a test can read the catalog and
481    /// open routes while the module is registered but not yet routable. A
482    /// blocking barrier because the HELLO handler is synchronous; the test
483    /// runs the HELLO on its own thread.
484    #[cfg(test)]
485    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
486}
487
488impl fmt::Debug for ControlHandler {
489    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
490        f.debug_struct("ControlHandler")
491            .field("registry", &self.registry)
492            .field("forwarding", &self.forwarding)
493            .field("process_liveness", &self.process_liveness.is_some())
494            .field("supervisor", &self.supervisor)
495            .field("subc_capabilities", &self.subc_capabilities)
496            .finish()
497    }
498}
499
500struct RouteOpenRequest {
501    target: RouteTarget,
502    identity: BindIdentity,
503    consumer_identity: Option<ConsumerIdentity>,
504    consumer_capabilities: Option<Vec<String>>,
505    role_versions: Option<BTreeMap<String, String>>,
506    busy_refusals: Option<bool>,
507    admission_facts: Option<serde_json::Value>,
508    scope: Option<ScopeSelector>,
509}
510
511#[derive(Clone, Copy)]
512enum RouteOpenPhase {
513    Admission,
514    ProjectRoot,
515    EgressReservation,
516    BindSend,
517    BindAckWait,
518}
519
520/// Store elapsed time for each route-open phase, including attempts that end
521/// in an early refusal or backend error. Admission also includes rechecks and
522/// commit bookkeeping performed between the other timed phases.
523struct RouteOpenTimings {
524    module_id: String,
525    principal: String,
526    admission: Duration,
527    project_root: Duration,
528    egress_reservation: Duration,
529    bind_send: Duration,
530    bind_ack_wait: Duration,
531    phase: Option<RouteOpenPhase>,
532    phase_started_at: Instant,
533}
534
535impl RouteOpenTimings {
536    fn new() -> Self {
537        Self {
538            module_id: String::new(),
539            // Refusals before attestation must not claim a verified principal.
540            principal: "unresolved".to_string(),
541            admission: Duration::ZERO,
542            project_root: Duration::ZERO,
543            egress_reservation: Duration::ZERO,
544            bind_send: Duration::ZERO,
545            bind_ack_wait: Duration::ZERO,
546            phase: Some(RouteOpenPhase::Admission),
547            phase_started_at: Instant::now(),
548        }
549    }
550
551    fn finish(&mut self) {
552        let elapsed = self.phase_started_at.elapsed();
553        match self.phase.take() {
554            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
555            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
556            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
557            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
558            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
559            None => {}
560        }
561    }
562
563    fn enter(&mut self, phase: RouteOpenPhase) {
564        self.finish();
565        self.phase = Some(phase);
566        self.phase_started_at = Instant::now();
567    }
568}
569
570struct RouteBindReservationGuard {
571    forwarding: Arc<ForwardingTable>,
572    endpoint: ModuleEndpointId,
573    relay_corr: u64,
574    armed: bool,
575}
576
577struct ModuleControlRpcGuard {
578    forwarding: Arc<ForwardingTable>,
579    endpoint: ModuleEndpointId,
580    corr: u64,
581    armed: bool,
582}
583
584impl ModuleControlRpcGuard {
585    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
586        Self {
587            forwarding,
588            endpoint,
589            corr,
590            armed: true,
591        }
592    }
593
594    fn disarm(&mut self) {
595        self.armed = false;
596    }
597}
598
599impl Drop for ModuleControlRpcGuard {
600    fn drop(&mut self) {
601        if self.armed {
602            let _ = self
603                .forwarding
604                .cancel_module_control_rpc(self.endpoint, self.corr);
605        }
606    }
607}
608
609impl RouteBindReservationGuard {
610    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
611        Self {
612            forwarding,
613            endpoint,
614            relay_corr,
615            armed: true,
616        }
617    }
618
619    fn release_and_disarm(&mut self) {
620        if !self.armed {
621            return;
622        }
623        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
624            self.endpoint,
625            self.relay_corr,
626            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
627        ) {
628            send_goodbye_target_best_effort(
629                &self.forwarding.counters(),
630                &target,
631                "canceled route.bind",
632            );
633        }
634        self.armed = false;
635    }
636
637    fn disarm(&mut self) {
638        self.armed = false;
639    }
640}
641
642impl Drop for RouteBindReservationGuard {
643    fn drop(&mut self) {
644        self.release_and_disarm();
645    }
646}
647
648/// Per-target-module circuit breaker around the `route.bind` relay.
649///
650/// The connection reader is serial per connection, so a module whose `on_bind`
651/// sits on the ack blocks every LATER frame on the connections that call it,
652/// including calls to unrelated modules. This does not make any module's bind
653/// fast; it stops the daemon paying the full budget again and again for a
654/// condition it has already observed.
655///
656/// State is keyed by TARGET MODULE and shared by every connection: a wedged
657/// module wedges everyone, so what one connection learned should protect the
658/// rest.
659///
660/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
661/// relay to that module has actually timed out, and is removed again when a
662/// relay is accepted or the module reconnects, so it cannot grow with traffic
663/// or with modules that behave.
664///
665/// # Why a `std` mutex here is not the head-of-line defect again
666///
667/// Acquisition never awaits. The critical section is a hash lookup plus a few
668/// integer updates, with no I/O and no `.await` inside it, so a reader task
669/// cannot be descheduled behind it the way it can behind
670/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
671/// primitive, held for the same kind of work, as the refusal counter this very
672/// path already increments.
673///
674/// It is also NOT on the data-plane splice path: only `route.open` and module
675/// registration touch it, so bound-route frames gain no state check and no
676/// contention.
677#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindBreakers {
679    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
680}
681
682#[derive(Debug, Clone, Default)]
683pub(crate) struct RouteBindConcurrency {
684    modules: Arc<Mutex<HashMap<String, usize>>>,
685}
686
687struct RouteBindConcurrencyGuard {
688    concurrency: RouteBindConcurrency,
689    module_id: String,
690}
691
692impl RouteBindConcurrency {
693    /// Admit without waiting. Waiting here would move the bind stall from the
694    /// module reply to a semaphore and restore reader head-of-line blocking.
695    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
696        let mut modules = self
697            .modules
698            .lock()
699            .expect("route.bind concurrency mutex poisoned");
700        let in_flight = modules.entry(module_id.to_string()).or_default();
701        if *in_flight >= limit {
702            return Err(*in_flight);
703        }
704        *in_flight += 1;
705        Ok(RouteBindConcurrencyGuard {
706            concurrency: self.clone(),
707            module_id: module_id.to_string(),
708        })
709    }
710}
711
712impl Drop for RouteBindConcurrencyGuard {
713    fn drop(&mut self) {
714        let mut modules = self
715            .concurrency
716            .modules
717            .lock()
718            .expect("route.bind concurrency mutex poisoned");
719        let remove = {
720            let in_flight = modules
721                .get_mut(&self.module_id)
722                .expect("admitted route.bind has a concurrency entry");
723            *in_flight -= 1;
724            *in_flight == 0
725        };
726        if remove {
727            modules.remove(&self.module_id);
728        }
729    }
730}
731
732#[derive(Debug, Default)]
733struct ModuleBreakerState {
734    /// Relay timeouts observed with no accepted relay in between.
735    consecutive_timeouts: u32,
736    /// `Some` while the breaker is open: the instant the cooldown expires and
737    /// the next arrival may probe. `None` means closed.
738    cooldown_until: Option<Instant>,
739    /// A half-open probe has been admitted and has not settled yet. This is
740    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
741    /// that read the cooldown, so concurrent opens arriving at the moment the
742    /// cooldown expires cannot all decide that they are the probe.
743    probe_in_flight: Option<Arc<()>>,
744}
745
746/// What the breaker decided for one `route.open`, before any relay work.
747enum RouteBindAdmission<'a> {
748    Admitted {
749        guard: RouteBindBreakerGuard<'a>,
750        /// This open is the single half-open probe, so the transition is worth
751        /// one log line.
752        probe: bool,
753    },
754    Refused {
755        consecutive_timeouts: u32,
756        /// What is left of the cooldown. Zero when the refusal is because the
757        /// one probe is already in flight rather than because the cooldown has
758        /// not elapsed.
759        retry_in: Duration,
760        probe_in_flight: bool,
761    },
762}
763
764/// An outstanding admission, which must be told how its relay settled.
765///
766/// `Drop` settles it as inconclusive, so an early return between admission and
767/// the relay -- or the whole handler being cancelled when the client
768/// disconnects -- releases a half-open probe slot instead of leaving the
769/// breaker wedged half-open with no further probes.
770struct RouteBindBreakerGuard<'a> {
771    breakers: RouteBindBreakers,
772    module_id: &'a str,
773    probe_token: Option<Arc<()>>,
774    settled: bool,
775}
776
777impl RouteBindBreakerGuard<'_> {
778    /// The module answered within the budget and took the bind. THE ONLY
779    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
780    /// breaker, which is a transition worth logging.
781    fn record_accepted(&mut self) -> bool {
782        self.settled = true;
783        self.breakers.record_accepted(self.module_id)
784    }
785
786    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
787    /// COUNTS TOWARD OPENING.
788    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
789        self.settled = true;
790        self.breakers.record_timeout(
791            self.module_id,
792            self.probe_token.as_ref(),
793            threshold,
794            cooldown,
795        )
796    }
797
798    /// Everything else: the module REJECTED the bind, its connection went away
799    /// mid-relay, or the waiter was cancelled.
800    ///
801    /// None of these is evidence that a module is slow, and each already has
802    /// its own refusal with its own code. A module that rejects a bind in
803    /// microseconds is healthy and must never be convicted for it; a module
804    /// that died has said nothing about the module that replaces it. So these
805    /// neither increment nor reset the count -- they only release a probe slot.
806    fn record_inconclusive(&mut self) {
807        self.settled = true;
808        self.breakers
809            .record_inconclusive(self.module_id, self.probe_token.as_ref());
810    }
811}
812
813impl Drop for RouteBindBreakerGuard<'_> {
814    fn drop(&mut self) {
815        if !self.settled {
816            self.breakers
817                .record_inconclusive(self.module_id, self.probe_token.as_ref());
818        }
819    }
820}
821
822/// The breaker moved to open, reported so the caller can log it outside the
823/// lock. Opening is rare and load-bearing; the refusals that follow are
824/// frequent and are counted rather than logged.
825struct BreakerOpened {
826    consecutive_timeouts: u32,
827    /// True when a failed probe re-opened an already-open breaker, which reads
828    /// very differently in a log from a first opening.
829    reopened_after_probe: bool,
830}
831
832impl RouteBindBreakers {
833    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
834        self.modules
835            .lock()
836            .expect("route.bind breaker mutex poisoned")
837    }
838
839    /// Decide whether this `route.open` may attempt its relay. Takes the map
840    /// lock and nothing else, and never awaits.
841    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
842        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
843            probe: probe_token.is_some(),
844            guard: RouteBindBreakerGuard {
845                breakers: self.clone(),
846                module_id,
847                probe_token,
848                settled: false,
849            },
850        };
851
852        let mut modules = self.lock();
853        let Some(state) = modules.get_mut(module_id) else {
854            return admitted(None);
855        };
856        let Some(cooldown_until) = state.cooldown_until else {
857            return admitted(None);
858        };
859        if state.probe_in_flight.is_some() {
860            return RouteBindAdmission::Refused {
861                consecutive_timeouts: state.consecutive_timeouts,
862                retry_in: Duration::ZERO,
863                probe_in_flight: true,
864            };
865        }
866        let now = Instant::now();
867        if now < cooldown_until {
868            return RouteBindAdmission::Refused {
869                consecutive_timeouts: state.consecutive_timeouts,
870                retry_in: cooldown_until - now,
871                probe_in_flight: false,
872            };
873        }
874        let token = Arc::new(());
875        state.probe_in_flight = Some(Arc::clone(&token));
876        admitted(Some(token))
877    }
878
879    fn record_accepted(&self, module_id: &str) -> bool {
880        self.lock()
881            .remove(module_id)
882            .is_some_and(|state| state.cooldown_until.is_some())
883    }
884
885    fn record_timeout(
886        &self,
887        module_id: &str,
888        probe_token: Option<&Arc<()>>,
889        threshold: u32,
890        cooldown: Duration,
891    ) -> Option<BreakerOpened> {
892        let mut modules = self.lock();
893        let state = modules.entry(module_id.to_string()).or_default();
894        let was_open = state.cooldown_until.is_some();
895        let was_probe = Self::owns_probe(state, probe_token);
896        if was_probe {
897            state.probe_in_flight = None;
898        }
899        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
900        if state.consecutive_timeouts < threshold {
901            return None;
902        }
903        state.cooldown_until = Some(Instant::now() + cooldown);
904        Some(BreakerOpened {
905            consecutive_timeouts: state.consecutive_timeouts,
906            reopened_after_probe: was_open && was_probe,
907        })
908    }
909
910    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
911        // A relay can finish after the breaker was reset or after it opened
912        // again and started a new probe. Only the guard whose token matches the
913        // active probe may release it, so a late relay never frees a newer probe.
914        state
915            .probe_in_flight
916            .as_ref()
917            .zip(token)
918            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
919    }
920
921    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
922        if let Some(state) = self.lock().get_mut(module_id) {
923            if Self::owns_probe(state, probe_token) {
924                state.probe_in_flight = None;
925            }
926        }
927    }
928
929    /// Discard what was learned about a module, because the process it was
930    /// learned about is gone. Returns the discarded count when it was non-zero.
931    ///
932    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
933    /// `module_id` is a configuration identity that outlives any particular
934    /// child; what the breaker observed was the process behind the module
935    /// connection of the moment. When a new connection registers under that id
936    /// the verdict's subject no longer exists, so the verdict is stale by
937    /// construction rather than merely likely to be wrong. Keeping it would
938    /// apply a dead process's record to a live one, which is the same defect
939    /// class this breaker exists to stop the daemon committing.
940    ///
941    /// A half-open probe in flight is discarded with the rest: it was a
942    /// question about the old process.
943    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
944        self.lock()
945            .remove(module_id)
946            .map(|state| state.consecutive_timeouts)
947            .filter(|discarded| *discarded > 0)
948    }
949
950    /// Open breakers, for the `server.describe` counters object. `None` when
951    /// none is open, so the key stays absent rather than present-and-empty.
952    ///
953    /// This is the operator's answer to "is this module refusing instantly or
954    /// is it fine?", which look identical from a client that retries and then
955    /// succeeds.
956    fn open_snapshot(&self) -> Option<serde_json::Value> {
957        let now = Instant::now();
958        let modules = self.lock();
959        let open = modules
960            .iter()
961            .filter_map(|(module_id, state)| {
962                let cooldown_until = state.cooldown_until?;
963                Some((
964                    module_id.clone(),
965                    serde_json::json!({
966                        "consecutive_timeouts": state.consecutive_timeouts,
967                        "cooldown_remaining_ms":
968                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
969                        "probe_in_flight": state.probe_in_flight.is_some(),
970                    }),
971                ))
972            })
973            .collect::<serde_json::Map<String, serde_json::Value>>();
974        (!open.is_empty()).then_some(serde_json::Value::Object(open))
975    }
976}
977
978impl ControlHandler {
979    pub fn new(registry: Arc<Registry>) -> Self {
980        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
981    }
982
983    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
984        let counters = forwarding.counters();
985        // Taken from the forwarding table rather than created here, so that the
986        // breaker a `route.open` consults is the same one a module's
987        // registration resets, however many handlers are built over one table.
988        let route_bind_breakers = forwarding.route_bind_breakers();
989        let route_bind_concurrency = forwarding.route_bind_concurrency();
990        let route_outages = forwarding.route_outages();
991        Self {
992            registry,
993            activity: None,
994            activity_streams: Arc::new(Mutex::new(HashMap::new())),
995            forwarding,
996            process_liveness: None,
997            supervisor: SupervisorHandle::new(),
998            subc_capabilities: Arc::from([
999                CAP_MANIFEST_REGISTRATION.to_string(),
1000                CAP_CHANNEL_LIFECYCLE.to_string(),
1001                CAP_PING_PONG.to_string(),
1002                CAP_SESSION_ATTACH.to_string(),
1003                CAP_ADMISSION_FACTS_RELAY.to_string(),
1004                CAP_SCOPES_V1.to_string(),
1005                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1006                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1007                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1008                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1009                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1010                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1011                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1012                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1013                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1014                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1015                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1016            ]),
1017            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1018            route_bind_relay_timeouts: BTreeMap::new(),
1019            route_bind_breakers,
1020            route_bind_concurrency,
1021            route_outages,
1022            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1023            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1024            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1025            storage_config: None,
1026            machine_id: None,
1027            admission_facts_carrier_module_id: None,
1028            admission_facts_targets: None,
1029            scopes: Arc::new(RwLock::new(ScopeTable::new(
1030                crate::daemon_config::default_scope_authority_owners(),
1031            ))),
1032            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1033            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1034            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1035            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1036            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1037            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1038            local_connections: Arc::new(Mutex::new(HashSet::new())),
1039            operator_app_policy: Default::default(),
1040            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1041            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1042            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1043            #[cfg(test)]
1044            app_open_test_platform: None,
1045            #[cfg(test)]
1046            app_open_test_launch: None,
1047            #[cfg(windows)]
1048            windows_verify: None,
1049            #[cfg(windows)]
1050            windows_flash: None,
1051            #[cfg(windows)]
1052            windows_track: None,
1053            #[cfg(all(test, windows))]
1054            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1055            #[cfg(test)]
1056            caller_lookup: None,
1057            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1058            rescan: None,
1059            connected_clients: ConnectedClients::new(),
1060            local_endpoint_unavailable: None,
1061            counters,
1062            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1063            daemon_shutdown: DaemonShutdownFlag::default(),
1064            daemon_provenance: DaemonProvenanceFacts::default(),
1065            #[cfg(test)]
1066            control_dispatch_delay: None,
1067            #[cfg(test)]
1068            provenance_probe_override: None,
1069            #[cfg(test)]
1070            route_bind_relay_sent_hold: None,
1071            #[cfg(test)]
1072            hello_registry_to_forwarding_hold: None,
1073        }
1074    }
1075
1076    /// Set the central storage policy: registering modules then receive their
1077    /// resolved storage descriptor in HELLO_ACK.
1078    pub fn with_storage_config(
1079        mut self,
1080        storage_config: Option<crate::daemon_config::StorageConfig>,
1081    ) -> Self {
1082        self.storage_config = storage_config;
1083        self
1084    }
1085
1086    /// Open the ledger on its own writer thread; an unavailable sink does not
1087    /// prevent the daemon from serving routing or supervisor requests.
1088    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1089        self.activity = Some(crate::activity::ActivityLedger::open(
1090            run_dir.join("activity.jsonl"),
1091            incarnation,
1092        ));
1093        self.supervisor.set_activity(self.activity.clone());
1094        self
1095    }
1096
1097    /// Set the machine id served to every registering module (HELLO_ACK) and on
1098    /// `server.describe`.
1099    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1100        self.machine_id = machine_id;
1101        self
1102    }
1103
1104    /// Configure the exact reserved module and target ids permitted to relay
1105    /// opaque admission facts. Config-file loading validates this authority;
1106    /// this builder keeps the same policy available to embedded test daemons.
1107    pub fn with_admission_facts_config(
1108        mut self,
1109        carrier_module_id: Option<String>,
1110        targets: Option<Vec<String>>,
1111    ) -> Self {
1112        self.admission_facts_carrier_module_id = carrier_module_id;
1113        self.admission_facts_targets = targets;
1114        self
1115    }
1116
1117    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1118    /// Replaces the scope table with an empty one under the new list, so call it
1119    /// while building the handler, before any module can sync.
1120    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1121        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1122        self.scope_authority_owners = owners;
1123        self
1124    }
1125
1126    /// Set the startup policy before installing the rescan context or serving requests.
1127    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1128        self.privileged_operator_prompt = mode;
1129        self
1130    }
1131
1132    /// Override the route.bind relay timeout. Used by tests that assert the
1133    /// timeout path so they don't block on the production-safe default.
1134    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1135        self.route_bind_relay_timeout = timeout;
1136        self
1137    }
1138
1139    /// Install per-module route.bind relay budget overrides. A module id
1140    /// listed here wins over the daemon-wide default set via
1141    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1142    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1143    /// the same `Duration` the bind path will use.
1144    pub fn with_route_bind_relay_timeouts(
1145        mut self,
1146        timeouts: impl IntoIterator<Item = (String, Duration)>,
1147    ) -> Self {
1148        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1149        self
1150    }
1151
1152    /// Resolve the route.bind relay budget for a specific target module id.
1153    /// Per-module overrides win; the daemon-wide value (set via
1154    /// `with_route_bind_relay_timeout` or the built-in default) is the
1155    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1156    /// the same resolution `handle_route_open` will use.
1157    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1158        self.route_bind_relay_timeouts
1159            .get(module_id)
1160            .copied()
1161            .unwrap_or(self.route_bind_relay_timeout)
1162    }
1163
1164    /// Override the per-module bind-relay breaker policy.
1165    ///
1166    /// Used by tests, which cannot spend three production budgets opening a
1167    /// breaker or twenty seconds waiting for its cooldown. The production
1168    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1169    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1170    /// reasoning for the numbers.
1171    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1172        self.route_bind_breaker_threshold = threshold.max(1);
1173        self.route_bind_breaker_cooldown = cooldown;
1174        self
1175    }
1176
1177    #[cfg(test)]
1178    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1179        self.health_probe_timeout = timeout;
1180        self
1181    }
1182
1183    #[cfg(test)]
1184    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1185        self.control_dispatch_delay = Some(delay);
1186        self
1187    }
1188
1189    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1190    /// announce it is parked, once to be released. Build the barrier for two
1191    /// parties, the test being the other one.
1192    #[cfg(test)]
1193    pub(crate) fn with_route_bind_relay_sent_hold(
1194        mut self,
1195        hold: Arc<tokio::sync::Barrier>,
1196    ) -> Self {
1197        self.route_bind_relay_sent_hold = Some(hold);
1198        self
1199    }
1200
1201    /// A HELLO waits on `hold` twice between reserving its registry entry and
1202    /// installing its forwarding endpoint: once to announce it is parked, once
1203    /// to be released. Build the barrier for two parties, the test being the
1204    /// other one, and run the HELLO on a thread of its own.
1205    #[cfg(test)]
1206    pub(crate) fn with_hello_registry_to_forwarding_hold(
1207        mut self,
1208        hold: Arc<std::sync::Barrier>,
1209    ) -> Self {
1210        self.hello_registry_to_forwarding_hold = Some(hold);
1211        self
1212    }
1213
1214    #[cfg(test)]
1215    fn hold_between_registry_and_forwarding(&self) {
1216        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1217            hold.wait();
1218            hold.wait();
1219        }
1220    }
1221
1222    pub fn with_process_liveness(
1223        mut self,
1224        process_liveness: Arc<dyn ModuleProcessLiveness>,
1225    ) -> Self {
1226        self.process_liveness = Some(process_liveness);
1227        self
1228    }
1229
1230    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1231        self.supervisor = supervisor;
1232        self.supervisor.set_activity(self.activity.clone());
1233        self
1234    }
1235
1236    pub fn with_daemon_provenance(
1237        mut self,
1238        pid: u32,
1239        started_at_ms: u64,
1240        executable_path: Option<PathBuf>,
1241        build_git_sha: Option<String>,
1242        build_lock_digest: Option<String>,
1243    ) -> Self {
1244        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1245        let process_start_time = process_start_time(pid);
1246        self.daemon_provenance = DaemonProvenanceFacts {
1247            build: DaemonBuildProvenance {
1248                build_git_sha,
1249                build_lock_digest,
1250            },
1251            pid: Some(pid),
1252            started_at_ms: Some(started_at_ms),
1253            start_clock: None,
1254            executable_path,
1255            executable_identity,
1256            process_start_time,
1257            probe: ExecutableIdentityProbe::default(),
1258        };
1259        self
1260    }
1261
1262    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1263        self.daemon_provenance.start_clock = Some(clock);
1264        self
1265    }
1266
1267    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1268    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1269        self.scopes
1270            .write()
1271            .unwrap_or_else(|poisoned| poisoned.into_inner())
1272            .set_wall_clock(Arc::new(clock));
1273        self
1274    }
1275
1276    #[cfg(test)]
1277    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1278        self.provenance_probe_override = Some(result);
1279        self
1280    }
1281
1282    /// Install the configured module set and its reserved capability bindings.
1283    /// Bindings are configuration-scoped and may point at a provider that has not
1284    /// been installed yet, so this does not require the bound module to exist.
1285    pub fn with_capability_config(
1286        self,
1287        modules: impl IntoIterator<Item = (String, bool)>,
1288        reserved_capabilities: BTreeMap<String, String>,
1289    ) -> Self {
1290        self.capability_evaluator
1291            .configure(modules, reserved_capabilities);
1292        self
1293    }
1294
1295    /// Share the supervisor's daemon-shutdown flag with this handler, so
1296    /// requirement changes after the daemon has begun its announced shutdown
1297    /// are not reported as faults.
1298    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1299        self.daemon_shutdown = flag;
1300        self
1301    }
1302
1303    pub fn with_supervisor_rescan(
1304        mut self,
1305        supervisor: Supervisor,
1306        config_path: impl Into<PathBuf>,
1307        configured_port: Option<u16>,
1308    ) -> Self {
1309        self.rescan = Some(SupervisorRescanContext {
1310            supervisor,
1311            config_path: config_path.into(),
1312            configured_port,
1313            storage_config: self.storage_config.clone(),
1314            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1315            admission_facts_targets: self.admission_facts_targets.clone(),
1316            scope_authority_owners: self.scope_authority_owners.clone(),
1317            privileged_operator_prompt: self.privileged_operator_prompt,
1318            generation: Arc::default(),
1319        });
1320        self
1321    }
1322
1323    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1324        self.connected_clients = connected_clients;
1325        self
1326    }
1327
1328    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1329    /// an operator can see a bind failure without reading the daemon log.
1330    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1331        self.local_endpoint_unavailable = reason;
1332        self
1333    }
1334
1335    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1336        Arc::clone(&self.forwarding)
1337    }
1338
1339    pub(crate) fn counters(&self) -> DaemonCounters {
1340        self.counters.clone()
1341    }
1342
1343    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1344    /// requirement event without depending on an operator polling a status command.
1345    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1346        tokio::spawn(async move {
1347            loop {
1348                self.capability_evaluator
1349                    .wait_for_change_or_deadline()
1350                    .await;
1351                self.refresh_capability_requirements();
1352            }
1353        });
1354    }
1355
1356    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1357    /// deadline passed during sleep is enforced on the first tick after wake.
1358    /// While no live scope carries a deadline the loop waits for an accepted
1359    /// scope change instead of ticking, so a daemon without deadlines never
1360    /// wakes for expiry.
1361    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1362        tokio::spawn(async move {
1363            loop {
1364                let has_deadlines = self
1365                    .scopes
1366                    .read()
1367                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1368                    .has_deadlines();
1369                if !has_deadlines {
1370                    self.scope_deadline_added.notified().await;
1371                    continue;
1372                }
1373                tokio::time::sleep(Duration::from_secs(1)).await;
1374                if let Err(error) = self.sweep_expired_scopes() {
1375                    warn!(%error, "scope expiry sweep failed");
1376                }
1377            }
1378        });
1379    }
1380
1381    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1382        let mut table = self
1383            .scopes
1384            .write()
1385            .unwrap_or_else(|poisoned| poisoned.into_inner());
1386        let swept = table.sweep_expired();
1387        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1388        drop(table);
1389        self.close_scope_drained_routes(drained);
1390        Ok(())
1391    }
1392
1393    fn publish_scope_changes(
1394        &self,
1395        changes: &[crate::scopes::ScopeTagChange],
1396        expired: &[crate::scopes::ScopeExpired],
1397    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1398        if changes.is_empty() {
1399            return Ok(Vec::new());
1400        }
1401        let drained = self
1402            .forwarding
1403            .publish_scope_changes(changes)
1404            .map_err(RouterError::Forwarding)?;
1405        let mut counts = HashMap::new();
1406        for route in &drained {
1407            *counts
1408                .entry((
1409                    route.scope.owner.as_str(),
1410                    route.scope.scope_ref.as_str(),
1411                    route.scope.tag.scope_epoch,
1412                ))
1413                .or_insert(0usize) += 1;
1414        }
1415        for scope in expired {
1416            let routes_closed = counts
1417                .get(&(
1418                    scope.owner.as_str(),
1419                    scope.scope_ref.as_str(),
1420                    scope.scope_epoch,
1421                ))
1422                .copied()
1423                .unwrap_or(0);
1424            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1425                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1426                routes_closed, "scope expired");
1427        }
1428        Ok(drained)
1429    }
1430
1431    fn runtime_capability_snapshot(
1432        &self,
1433    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1434        let runtime = self
1435            .supervisor
1436            .list()
1437            .into_iter()
1438            .map(|module| {
1439                let status = module.status().map_err(|err| {
1440                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1441                })?;
1442                Ok(RuntimeModule {
1443                    module_id: status.module_id,
1444                    state: status.state,
1445                    enabled: status.enabled,
1446                })
1447            })
1448            .collect::<Result<Vec<_>, RouterError>>()?;
1449        // Include every registration, published or not. The capability
1450        // evaluator counts a module as providing its capabilities as soon as it
1451        // has registered, even before `catalog.list` shows it, so a HELLO in
1452        // progress does not change other modules' required-capability
1453        // verdicts. See `Registry::list_registered_modules`.
1454        let registrations = self.registry.list_registered_modules().map_err(|err| {
1455            RouterError::backend(
1456                0,
1457                0,
1458                format!("failed to list capability registrations: {err}"),
1459            )
1460        })?;
1461        let registrations = registrations
1462            .into_iter()
1463            .map(|registration| RegisteredModule {
1464                module_id: registration.manifest.module_id,
1465                module_version: registration.manifest.module_version,
1466                capabilities: registration.manifest.capabilities,
1467            })
1468            .collect();
1469        Ok((runtime, registrations))
1470    }
1471
1472    /// The capability side effects of a module becoming the active registration
1473    /// for its id: cache its manifest (logging how it differs from the cached
1474    /// one), run the deny census when its declarations call for one, and recompute the
1475    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1476    /// candidate's does not, and the supervisor does it at promotion instead,
1477    /// through [`crate::supervise::SwapPromotionObserver`].
1478    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1479        let cached_registration = RegisteredModule {
1480            module_id: registration.manifest.module_id.clone(),
1481            module_version: registration.manifest.module_version.clone(),
1482            capabilities: registration.manifest.capabilities.clone(),
1483        };
1484        let change = self.capability_evaluator.record_hello(&cached_registration);
1485        log_manifest_change(&cached_registration.module_id, &change);
1486        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1487            self.enforce_capability_denies();
1488        }
1489        self.refresh_capability_requirements();
1490    }
1491
1492    /// Point the shared supervisor handle at this handler for swap promotions.
1493    /// Called wherever a handler is put behind the `Arc` the router serves, so
1494    /// it can be held weakly.
1495    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1496        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1497            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1498        self.supervisor.set_swap_promotion_observer(observer);
1499    }
1500
1501    pub fn refresh_capability_requirements(&self) {
1502        match self.runtime_capability_snapshot() {
1503            Ok((runtime, registrations)) => {
1504                log_requirement_events(
1505                    self.capability_evaluator
1506                        .evaluate_now(&runtime, &registrations),
1507                    &self.daemon_shutdown,
1508                );
1509            }
1510            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1511        }
1512    }
1513
1514    /// Reconcile only live, attested route bindings after a capability deny edge
1515    /// or target claim was added. This is deliberately a control-plane census:
1516    /// the opaque forwarding hot path must not grow a per-frame capability check.
1517    fn enforce_capability_denies(&self) {
1518        let registrations = match self.registry.list_registered_modules() {
1519            Ok(snapshot) => snapshot,
1520            Err(err) => {
1521                warn!(error = %err, "failed to read registrations for capability deny census");
1522                return;
1523            }
1524        };
1525        let manifests = registrations
1526            .into_iter()
1527            .map(|registration| {
1528                (
1529                    registration.manifest.module_id.clone(),
1530                    registration.manifest,
1531                )
1532            })
1533            .collect::<BTreeMap<_, _>>();
1534        let census = match self.forwarding.route_census(None) {
1535            Ok(census) => census,
1536            Err(err) => {
1537                warn!(error = %err, "failed to read route census for capability deny enforcement");
1538                return;
1539            }
1540        };
1541
1542        for (target_module_id, routes) in census {
1543            let Some(target_manifest) = manifests.get(&target_module_id) else {
1544                continue;
1545            };
1546            let mut closed_routes = Vec::new();
1547            let mut goodbyes = Vec::new();
1548            for route in routes {
1549                let Principal::Reserved {
1550                    module_id: opening_module_id,
1551                } = &route.principal
1552                else {
1553                    continue;
1554                };
1555                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1556                    continue;
1557                };
1558                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1559                    continue;
1560                };
1561
1562                match self.forwarding.release_client_route(
1563                    route.goodbye_target.connection_id,
1564                    route.goodbye_target.channel,
1565                    route.goodbye_target.epoch,
1566                ) {
1567                    Ok(RouteRelease::Removed(module_goodbye)) => {
1568                        warn!(
1569                            opening_module_id,
1570                            target_module_id,
1571                            capability,
1572                            "force-closing route because an attested capability deny edge now matches"
1573                        );
1574                        goodbyes.push(module_goodbye);
1575                        goodbyes.push(route.goodbye_target.clone());
1576                        closed_routes.push(route);
1577                    }
1578                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1579                    Err(err) => warn!(
1580                        opening_module_id,
1581                        target_module_id,
1582                        capability,
1583                        error = %err,
1584                        "failed to force-close capability-denied route"
1585                    ),
1586                }
1587            }
1588
1589            if closed_routes.is_empty() {
1590                continue;
1591            }
1592            send_route_control_pushes(
1593                &self.forwarding,
1594                closed_routes,
1595                ClientControlPush::RouteClosed {
1596                    module_id: target_module_id,
1597                    channels: Vec::new(),
1598                    reason: RouteCloseReason::CapabilityDenied,
1599                    drained: false,
1600                    abandoned: 0,
1601                    excluded_subscriptions: 0,
1602                    terminal: Some(false),
1603                },
1604            );
1605            // route.closed records the reason; the route GOODBYE ends the
1606            // client's handle and settles any calls still pending on it.
1607            self.emit_route_goodbyes(goodbyes);
1608        }
1609    }
1610
1611    /// Why a registered module is not accepting new route binds, or `None` when
1612    /// it is. This is the module's effective readiness: its declared readiness
1613    /// first, then every `need: required` capability it declares evaluating to
1614    /// `provided`. `route.open` and `catalog.list` both read it here so the
1615    /// catalog never reports a module routable that `route.open` would refuse.
1616    fn not_ready_reason(
1617        &self,
1618        registration: &crate::registry::ModuleRegistration,
1619    ) -> Option<NotReadyReason> {
1620        if !registration.ready {
1621            return Some(NotReadyReason {
1622                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1623                capability: None,
1624            });
1625        }
1626        self.first_unprovided_required_capability(registration)
1627            .map(|capability| NotReadyReason {
1628                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1629                capability: Some(capability),
1630            })
1631    }
1632
1633    /// The lexicographically first capability this registration declares
1634    /// `need: required` whose evaluator verdict is not `provided`.
1635    ///
1636    /// The verdicts are the capability evaluator's own; nothing here decides
1637    /// what "provided" means. The evaluator counts a capability provided as
1638    /// soon as a module claiming it has REGISTERED, not once that module is
1639    /// ready. That distinction is what keeps two modules that require each
1640    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1641    /// is ready", each would wait for the other to become ready first and
1642    /// neither ever would. Do not tighten it to readiness.
1643    ///
1644    /// A required capability with no verdict at all means this registration's
1645    /// HELLO or catalog.update landed after the last recompute; recompute once
1646    /// rather than let a missing verdict read as either answer. If it is still
1647    /// missing (the recompute itself failed) the capability counts as
1648    /// unprovided: the refusal is retryable, and routing a module whose
1649    /// required provider is unknown is the outcome this check exists to stop.
1650    fn first_unprovided_required_capability(
1651        &self,
1652        registration: &crate::registry::ModuleRegistration,
1653    ) -> Option<String> {
1654        let required = registration
1655            .manifest
1656            .capabilities
1657            .iter()
1658            .flat_map(|declarations| declarations.requires.iter())
1659            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1660            .map(|requirement| requirement.capability.as_str())
1661            .collect::<BTreeSet<_>>();
1662        if required.is_empty() {
1663            return None;
1664        }
1665        let module_id = registration.manifest.module_id.as_str();
1666        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1667        if required
1668            .iter()
1669            .any(|capability| verdict(capability).is_none())
1670        {
1671            self.refresh_capability_requirements();
1672        }
1673        required
1674            .into_iter()
1675            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1676            .map(str::to_string)
1677    }
1678
1679    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1680        self.capability_evaluator
1681            .statuses()
1682            .into_iter()
1683            .map(capability_requirement_status)
1684            .collect()
1685    }
1686
1687    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1688    /// registration-release watch. The signal is what the supervisor waits on
1689    /// before spawning a replacement, so it must only fire once forwarding
1690    /// teardown is also done (see [`Self::cleanup_connection`] /
1691    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1692    /// admitted but its forwarding endpoint could not be installed.
1693    fn deregister_connection(
1694        &self,
1695        connection_id: ConnectionId,
1696        reason: RegistrationEndReason,
1697    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1698        self.registry
1699            .deregister_connection_with_reason(connection_id, reason)
1700    }
1701
1702    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1703        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1704            return None;
1705        }
1706        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1707            parse_client_control_request(&frame.body)
1708        else {
1709            return None;
1710        };
1711        Some(target_module_id(&target).to_string())
1712    }
1713
1714    pub(crate) fn route_open_capacity_refusal(
1715        &self,
1716        ctx: &RouteCtx,
1717        frame: &Frame,
1718        target_module_id: &str,
1719        in_flight: usize,
1720        limit: usize,
1721    ) -> Result<Frame, RouterError> {
1722        // Sent before the request reaches `handle_route_open`, so the caller
1723        // fields come from the raw frame here.
1724        let caller = self.route_open_caller_from_frame(frame);
1725        self.route_open_admission_refusal_frame(
1726            ctx,
1727            &caller,
1728            frame,
1729            target_module_id,
1730            "open_admission_full",
1731            (in_flight, limit),
1732            format!(
1733                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1734            ),
1735        )
1736    }
1737
1738    fn route_open_target_capacity_refusal(
1739        &self,
1740        ctx: &RouteCtx,
1741        caller: &RouteOpenCaller,
1742        frame: &Frame,
1743        target_module_id: &str,
1744        in_flight: usize,
1745    ) -> Result<Frame, RouterError> {
1746        self.route_open_admission_refusal_frame(
1747            ctx,
1748            caller,
1749            frame,
1750            target_module_id,
1751            "target_binds_full",
1752            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1753            format!(
1754                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1755            ),
1756        )
1757    }
1758
1759    /// Admission pressure clears as existing binds settle, so its refusal must
1760    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1761    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1762    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1763    /// cannot currently reach its target; `module_timeout` would falsely claim
1764    /// that a wait expired. A new, cleaner code would be terminal to deployed
1765    /// clients, so it requires a client-tolerance rollout before daemon emission.
1766    #[allow(clippy::too_many_arguments)]
1767    fn route_open_admission_refusal_frame(
1768        &self,
1769        ctx: &RouteCtx,
1770        caller: &RouteOpenCaller,
1771        frame: &Frame,
1772        target_module_id: &str,
1773        reason: &'static str,
1774        (in_flight, limit): (usize, usize),
1775        message: impl Into<String>,
1776    ) -> Result<Frame, RouterError> {
1777        let code = error_codes::TARGET_UNAVAILABLE;
1778        self.counters.increment_route_open_refused(code);
1779        info!(
1780            target: "control",
1781            code,
1782            reason,
1783            module_id = ?target_module_id,
1784            connection_id = ctx.connection_id.get(),
1785            principal = %caller.principal,
1786            scope = %caller.scope,
1787            harness = %caller.harness,
1788            in_flight,
1789            limit,
1790            "route.open refused"
1791        );
1792        control_error_frame(frame, code, message.into())
1793    }
1794
1795    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1796    ///
1797    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1798    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1799    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1800    #[cfg(test)]
1801    pub fn handle_control(
1802        &self,
1803        connection_id: ConnectionId,
1804        frame: Frame,
1805    ) -> Result<Vec<Frame>, RouterError> {
1806        match frame.header.ty {
1807            FrameType::Ping => Ok(vec![pong(&frame)?]),
1808            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1809            FrameType::Goodbye => self.handle_goodbye(connection_id),
1810            ty => Ok(vec![control_error_frame(
1811                &frame,
1812                "unsupported_control_frame",
1813                format!("unsupported channel-0 frame {ty:?}"),
1814            )?]),
1815        }
1816    }
1817
1818    pub async fn handle_control_frame(
1819        &self,
1820        ctx: &RouteCtx,
1821        frame: Frame,
1822    ) -> Result<Vec<Frame>, RouterError> {
1823        self.handle_control_frame_timed(ctx, frame, None).await
1824    }
1825
1826    #[cfg(any(test, feature = "test-support"))]
1827    pub fn with_test_operator_app_policy(
1828        mut self,
1829        policy: crate::operator_apps::TestOperatorAppPolicy,
1830    ) -> Self {
1831        self.operator_app_policy = policy.into();
1832        self
1833    }
1834
1835    /// Inject the Windows signature check for tests. The injected check
1836    /// answers the verified version string or the rejection reason; the
1837    /// daemon enforces the floor itself on the returned version.
1838    #[cfg(all(windows, any(test, feature = "test-support")))]
1839    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1840        self.windows_verify = Some(verify);
1841        self
1842    }
1843
1844    /// Record which pids the daemon would flash for `already_running`.
1845    #[cfg(all(windows, any(test, feature = "test-support")))]
1846    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1847        self.windows_flash = Some(flash);
1848        self
1849    }
1850
1851    /// Inject a tracking failure for tests: makes `track_launched_process`
1852    /// fail so a test can prove a launch that cannot be tracked is stopped,
1853    /// never left running.
1854    #[cfg(all(windows, any(test, feature = "test-support")))]
1855    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1856        self.windows_track = Some(track);
1857        self
1858    }
1859
1860    pub(crate) fn record_captured_peer(
1861        &self,
1862        connection: ConnectionId,
1863        peer: subc_os::local_peer::CapturedPeer,
1864    ) {
1865        self.record_observed_peer(connection, peer.observation().clone());
1866        #[cfg(windows)]
1867        if let Some(stamp) = self.stamp_windows_launch(&peer) {
1868            self.operator_apps
1869                .lock()
1870                .unwrap_or_else(|p| p.into_inner())
1871                .insert(connection, stamp);
1872        }
1873        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1874            self.operator_apps
1875                .lock()
1876                .unwrap_or_else(|p| p.into_inner())
1877                .insert(connection, stamp);
1878        }
1879    }
1880
1881    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1882        self.local_connections
1883            .lock()
1884            .unwrap_or_else(|p| p.into_inner())
1885            .insert(connection);
1886    }
1887
1888    // Keep the connection-level delivery decision shared with route.bind. The
1889    // target's capability opt-in is separate from recognition of the caller.
1890    fn connection_operator_app(
1891        &self,
1892        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1893    ) -> Option<subc_protocol::peer::OperatorApp> {
1894        stamp
1895            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1896            .map(|stamp| stamp.app.clone())
1897    }
1898
1899    fn handle_connection_describe(
1900        &self,
1901        ctx: &RouteCtx,
1902        frame: Frame,
1903    ) -> Result<Vec<Frame>, RouterError> {
1904        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1905            return Ok(vec![control_error_frame(
1906                &frame,
1907                "invalid_control_body",
1908                "malformed connection.describe body",
1909            )?]);
1910        }
1911        let local = self
1912            .local_connections
1913            .lock()
1914            .unwrap_or_else(|p| p.into_inner())
1915            .contains(&ctx.connection_id);
1916        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1917        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1918        Ok(vec![control_response_body_frame(
1919            &frame,
1920            &subc_control::ConnectionDescribeReply::new(local, operator_app),
1921            ops::CONNECTION_DESCRIBE,
1922        )?])
1923    }
1924
1925    pub(crate) fn record_observed_peer(
1926        &self,
1927        connection: ConnectionId,
1928        peer: subc_os::local_peer::ObservedPeer,
1929    ) {
1930        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1931        let peer = match peer {
1932            OsPeer::Unobserved => ObservedPeer::unobserved(),
1933            OsPeer::Observed(peer) => {
1934                let observed = match peer.user {
1935                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1936                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1937                };
1938                observed
1939                    .with_exe(
1940                        peer.exe
1941                            .and_then(|path| path.into_os_string().into_string().ok()),
1942                    )
1943                    .with_signing_id(peer.signing_id)
1944            }
1945        };
1946        self.observed_peers
1947            .lock()
1948            .unwrap_or_else(|p| p.into_inner())
1949            .insert(connection, peer);
1950    }
1951
1952    pub(crate) fn record_caller_endpoints(
1953        &self,
1954        connection: ConnectionId,
1955        endpoints: Option<(SocketAddr, SocketAddr)>,
1956    ) {
1957        let mut callers = self
1958            .caller_endpoints
1959            .lock()
1960            .unwrap_or_else(|p| p.into_inner());
1961        callers.remove(&connection);
1962        if let Some(endpoints) = endpoints {
1963            callers.insert(connection, endpoints);
1964        }
1965    }
1966
1967    #[cfg(test)]
1968    pub(crate) fn with_caller_lookup_for_test(
1969        mut self,
1970        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1971    ) -> Self {
1972        self.caller_lookup = Some(Arc::new(lookup));
1973        self
1974    }
1975
1976    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1977        let endpoints = self
1978            .caller_endpoints
1979            .lock()
1980            .unwrap_or_else(|p| p.into_inner())
1981            .get(&connection)
1982            .copied();
1983        #[cfg(test)]
1984        let lookup = self.caller_lookup.clone();
1985        tokio::task::spawn_blocking(move || {
1986            #[cfg(test)]
1987            if let Some(lookup) = lookup {
1988                return lookup(endpoints);
1989            }
1990            endpoints
1991                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1992                .unwrap_or_default()
1993        })
1994        .await
1995        .unwrap_or_default()
1996    }
1997
1998    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1999        frame.header.channel == 0
2000            && frame.header.ty == FrameType::Request
2001            && self
2002                .forwarding
2003                .module_endpoint_for_connection(ctx.connection_id)
2004                .is_ok_and(|endpoint| endpoint.is_none())
2005            && parse_client_control_request(&frame.body)
2006                .ok()
2007                .is_some_and(|request| privileged_operation(&request).is_some())
2008    }
2009
2010    pub(crate) async fn handle_control_frame_timed(
2011        &self,
2012        ctx: &RouteCtx,
2013        frame: Frame,
2014        dispatch_started_at: Option<StdInstant>,
2015    ) -> Result<Vec<Frame>, RouterError> {
2016        match frame.header.ty {
2017            FrameType::Ping => Ok(vec![pong(&frame)?]),
2018            FrameType::Hello => {
2019                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2020            }
2021            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2022            FrameType::Cancel => {
2023                // A Cancel on channel 0 names a waiting operator.confirm or a
2024                // subscription. Successful cancellation has no ordinary reply.
2025                if self
2026                    .forwarding
2027                    .operator_confirms()
2028                    .cancel(ctx.connection_id, frame.header.corr)
2029                    || self
2030                        .supervisor
2031                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2032                    || self.cancel_activity_subscription(ctx, &frame)?
2033                {
2034                    Ok(Vec::new())
2035                } else {
2036                    Ok(vec![control_error_frame(
2037                        &frame,
2038                        "unknown_subscription",
2039                        "no subscription has this correlation id",
2040                    )?])
2041                }
2042            }
2043            FrameType::Request => {
2044                // This additive operation is not part of the existing exhaustive
2045                // module-control enum. Probe the op before decoding that enum.
2046                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2047                if op
2048                    .as_ref()
2049                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2050                {
2051                    return self.handle_activity_record(ctx, frame).await;
2052                }
2053                if op.as_ref().is_some_and(|probe| {
2054                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2055                }) {
2056                    return self.handle_activity_read(ctx, frame).await;
2057                }
2058                if op
2059                    .as_ref()
2060                    .is_some_and(|probe| probe.op == "operator.confirm")
2061                {
2062                    return self.handle_operator_confirm(ctx, frame);
2063                }
2064                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2065                    return self
2066                        .observe_control_wait(
2067                            ctx,
2068                            None,
2069                            ops::APP_OPEN,
2070                            self.handle_app_open(ctx, frame),
2071                        )
2072                        .await;
2073                }
2074                if op
2075                    .as_ref()
2076                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2077                {
2078                    return self.handle_connection_describe(ctx, frame);
2079                }
2080                if op
2081                    .as_ref()
2082                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2083                {
2084                    return self.handle_scope_served_by(ctx.connection_id, frame);
2085                }
2086                if self
2087                    .forwarding
2088                    .module_endpoint_for_connection(ctx.connection_id)
2089                    .map_err(RouterError::Forwarding)?
2090                    .is_some()
2091                {
2092                    if !is_known_module_request_op(&frame.body) {
2093                        return Ok(vec![control_error_frame(
2094                            &frame,
2095                            "unsupported_control_frame",
2096                            "module-originated channel-0 REQUEST is not supported",
2097                        )?]);
2098                    }
2099                    let request = match parse_module_control_request_from_module(&frame.body) {
2100                        Ok(request) => request,
2101                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2102                            return Ok(vec![control_error_frame(
2103                                &frame,
2104                                "unsupported_control_frame",
2105                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2106                            )?])
2107                        }
2108                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2109                            return Ok(vec![control_error_frame(
2110                                &frame,
2111                                "invalid_control_body",
2112                                format!("malformed module control body: {err}"),
2113                            )?])
2114                        }
2115                    };
2116                    let op = module_control_request_op(&request);
2117                    let corr = frame.header.corr;
2118                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2119                    let result =
2120                        self.handle_module_control_request(ctx.connection_id, frame, request);
2121                    log_slow_control_dispatch(
2122                        dispatch_started_at,
2123                        op,
2124                        ctx.connection_id,
2125                        corr,
2126                        None,
2127                    );
2128                    return result;
2129                }
2130
2131                if is_known_module_request_op(&frame.body) {
2132                    return Ok(vec![control_error_frame(
2133                        &frame,
2134                        "not_registered",
2135                        "catalog.update requires an active module registration owned by this connection",
2136                    )?]);
2137                }
2138
2139                let request = match parse_client_control_request(&frame.body) {
2140                    Ok(request) => request,
2141                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2142                        return Ok(vec![control_error_frame(
2143                            &frame,
2144                            "unknown_control_op",
2145                            format!("unknown client control op: {err}"),
2146                        )?])
2147                    }
2148                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2149                        return Ok(vec![control_error_frame(
2150                            &frame,
2151                            "invalid_control_body",
2152                            format!("malformed client control body: {err}"),
2153                        )?])
2154                    }
2155                };
2156                let op = client_control_request_op(&request);
2157                let corr = frame.header.corr;
2158                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2159                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2160                #[cfg(test)]
2161                if let Some(delay) = self.control_dispatch_delay {
2162                    tokio::time::sleep(delay).await;
2163                }
2164                let result = self
2165                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2166                    .await;
2167                if let Some(timings) = route_open_timings.as_mut() {
2168                    timings.finish();
2169                }
2170                log_slow_control_dispatch(
2171                    dispatch_started_at,
2172                    op,
2173                    ctx.connection_id,
2174                    corr,
2175                    route_open_timings.as_ref(),
2176                );
2177                result
2178            }
2179            FrameType::Push => {
2180                let Some(endpoint) = self
2181                    .forwarding
2182                    .module_endpoint_for_connection(ctx.connection_id)
2183                    .map_err(RouterError::Forwarding)?
2184                else {
2185                    return Ok(vec![control_error_frame(
2186                        &frame,
2187                        "unsupported_control_frame",
2188                        "client-originated channel-0 PUSH is not supported",
2189                    )?]);
2190                };
2191                self.handle_status_update(endpoint, frame)
2192            }
2193            FrameType::Response | FrameType::Error
2194                if self
2195                    .forwarding
2196                    .module_endpoint_for_connection(ctx.connection_id)
2197                    .map_err(RouterError::Forwarding)?
2198                    .is_some() =>
2199            {
2200                self.handle_module_relay_response(ctx.connection_id, frame)
2201            }
2202            ty => Ok(vec![control_error_frame(
2203                &frame,
2204                "unsupported_control_frame",
2205                format!("unsupported channel-0 frame {ty:?}"),
2206            )?]),
2207        }
2208    }
2209
2210    pub fn cleanup_connection(
2211        &self,
2212        connection_id: ConnectionId,
2213    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2214        self.local_connections
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner())
2217            .remove(&connection_id);
2218        self.caller_endpoints
2219            .lock()
2220            .unwrap_or_else(|p| p.into_inner())
2221            .remove(&connection_id);
2222        self.build_class_warned
2223            .lock()
2224            .unwrap_or_else(|p| p.into_inner())
2225            .remove(&connection_id);
2226        self.observed_peers
2227            .lock()
2228            .unwrap_or_else(|p| p.into_inner())
2229            .remove(&connection_id);
2230        self.operator_apps
2231            .lock()
2232            .unwrap_or_else(|p| p.into_inner())
2233            .remove(&connection_id);
2234        self.forwarding
2235            .operator_confirms()
2236            .connection_closed(connection_id);
2237        self.cleanup_connection_with_end_reason(
2238            connection_id,
2239            RegistrationEndReason::ConnectionClosed,
2240        )
2241    }
2242
2243    fn cleanup_connection_with_end_reason(
2244        &self,
2245        connection_id: ConnectionId,
2246        requested_reason: RegistrationEndReason,
2247    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2248        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2249            self.registry
2250                .get_module_by_connection(connection_id)?
2251                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2252                .and_then(|module| module.registration_end_reason().ok().flatten())
2253                .unwrap_or(requested_reason)
2254        } else {
2255            requested_reason
2256        };
2257        let crash_closed = self
2258            .registry
2259            .get_module_by_connection(connection_id)?
2260            .and_then(|registration| {
2261                self.forwarding
2262                    .module_endpoint_for_connection(connection_id)
2263                    .ok()
2264                    .flatten()
2265                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2266                    .map(|routes| (registration.manifest.module_id, routes))
2267            });
2268        let crash_closed = crash_closed.map(|(module_id, routes)| {
2269            let terminal = match self.supervisor.get(&module_id) {
2270                None => false,
2271                Some(module) => match module.will_recover_after_connection_loss() {
2272                    Ok(will_recover) => !will_recover,
2273                    Err(err) => {
2274                        warn!(
2275                            %module_id,
2276                            error = %err,
2277                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2278                        );
2279                        false
2280                    }
2281                },
2282            };
2283            // The forwarding table gates all providers at the start of daemon
2284            // shutdown, before their connections are closed. An ordinary
2285            // module disconnect still reports crash if that gate is not set.
2286            let (reason, drained) = match self
2287                .forwarding
2288                .daemon_drain_result_for_connection(connection_id)
2289            {
2290                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2291                Ok(None) => (RouteCloseReason::Crash, false),
2292                Err(err) => {
2293                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2294                    (RouteCloseReason::Crash, false)
2295                }
2296            };
2297            (module_id, routes, reason, drained, terminal)
2298        });
2299        let registrations = self.deregister_connection(connection_id, end_reason);
2300        let cleanup = if crash_closed.is_some() {
2301            self.forwarding.cleanup_connection_counted(connection_id)
2302        } else {
2303            // A module connection's teardown needs the count of abandoned
2304            // route.bind relays for its route.closed notice below. Any other
2305            // connection, such as a client's, sends no such notice and needs
2306            // only its routes released, so it uses the route-only wrapper and
2307            // reports zero.
2308            self.forwarding
2309                .cleanup_connection(connection_id)
2310                .map(|released| crate::forwarding::ConnectionCleanup {
2311                    released,
2312                    abandoned_relays: 0,
2313                })
2314        };
2315        // The route.closed push waits for forwarding teardown because only
2316        // teardown knows how many pending route.bind relays it aborted. It still
2317        // goes out before the GOODBYEs for the released routes, and its targets
2318        // were captured above, before teardown removed those routes.
2319        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2320            let abandoned = cleanup
2321                .as_ref()
2322                .map_or(0, |cleanup| cleanup.abandoned_relays);
2323            send_route_control_pushes(
2324                &self.forwarding,
2325                routes,
2326                ClientControlPush::RouteClosed {
2327                    module_id,
2328                    channels: Vec::new(),
2329                    reason,
2330                    drained,
2331                    abandoned,
2332                    excluded_subscriptions: 0,
2333                    terminal: Some(terminal),
2334                },
2335            );
2336        }
2337        if let Ok(cleanup) = cleanup {
2338            self.emit_route_goodbyes(cleanup.released);
2339        }
2340        // Signal the registration-release watch only now that BOTH registry and
2341        // forwarding teardown are done, so a supervisor waiting to spawn a
2342        // replacement never observes release while old routes still exist.
2343        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2344            crate::supervise::notify_registration_release();
2345            self.capability_evaluator.wake_deadline_loop();
2346            self.refresh_capability_requirements();
2347        }
2348        self.supervisor.remove_spawn_subscribers(connection_id);
2349        self.activity_streams
2350            .lock()
2351            .unwrap_or_else(|p| p.into_inner())
2352            .retain(|(id, _), task| {
2353                if *id == connection_id {
2354                    task.abort();
2355                    false
2356                } else {
2357                    true
2358                }
2359            });
2360        // Sync authority dies with its connection, so the owner's next
2361        // connection can take it; the owner's scopes stay as they are.
2362        self.hello_launch_nonces
2363            .lock()
2364            .unwrap_or_else(|poisoned| poisoned.into_inner())
2365            .forget(connection_id);
2366        self.scopes
2367            .write()
2368            .unwrap_or_else(|poisoned| poisoned.into_inner())
2369            .release_connection(connection_id);
2370        registrations
2371    }
2372
2373    pub(crate) fn handle_route_goodbye(
2374        &self,
2375        connection_id: ConnectionId,
2376        route_channel: u16,
2377        route_epoch: u32,
2378    ) -> Result<bool, RouterError> {
2379        debug!(
2380            connection_id = connection_id.get(),
2381            route_channel, route_epoch, "handling route GOODBYE"
2382        );
2383        let RouteRelease::Removed(released_route) = self
2384            .forwarding
2385            .release_client_route(connection_id, route_channel, route_epoch)
2386            .map_err(RouterError::Forwarding)?
2387        else {
2388            return Ok(false);
2389        };
2390        self.emit_route_goodbyes(vec![released_route]);
2391        Ok(true)
2392    }
2393
2394    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2395        for released in released_routes {
2396            let frame = match Frame::build_with_version(
2397                released.negotiated_ver,
2398                FrameType::Goodbye,
2399                control_flags(),
2400                released.channel,
2401                released.epoch,
2402                0,
2403                Vec::new(),
2404            ) {
2405                Ok(frame) => frame,
2406                Err(err) => {
2407                    warn!(
2408                        route_channel = released.channel,
2409                        error = %err,
2410                        "failed to build route GOODBYE frame"
2411                    );
2412                    continue;
2413                }
2414            };
2415            if !released.close_on_delivery_failure() {
2416                crate::forwarding::send_module_route_goodbye(
2417                    &self.counters,
2418                    &released.sink,
2419                    frame,
2420                    released.module_id.as_deref(),
2421                    "client route released",
2422                );
2423                continue;
2424            }
2425            if let Err(err) = released.sink.try_send(frame) {
2426                warn!(
2427                    target_connection_id = released.connection_id.get(),
2428                    route_channel = released.channel,
2429                    error = %err,
2430                    "route GOODBYE was not delivered to client; closing target connection"
2431                );
2432                if self
2433                    .forwarding
2434                    .escalate_client_delivery_failure(
2435                        released.connection_id,
2436                        released.channel,
2437                        released.epoch,
2438                        CloseReason::new(
2439                            "route_goodbye_delivery_failed",
2440                            format!(
2441                                "failed to enqueue route GOODBYE for channel {}: {err}",
2442                                released.channel
2443                            ),
2444                        ),
2445                        crate::forwarding::UndeliveredFrame {
2446                            module_id: released.module_id.as_deref(),
2447                            sink: &released.sink,
2448                        },
2449                    )
2450                    .unwrap_or(false)
2451                {
2452                    self.counters.increment_goodbye_relay_client_failed();
2453                }
2454            }
2455        }
2456    }
2457
2458    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2459    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2460    /// own commit failed after the module had already accepted). Without this, a
2461    /// module that accepts late keeps a binding subc has torn down, so a later
2462    /// frame on that module channel could misdeliver if the channel is reused.
2463    ///
2464    /// Never closes the shared module connection on failure: a dropped notification
2465    /// only wastes a bounded amount of warm module-side state, which the module's
2466    /// own idle reaper reclaims. Only call this once the route.bind relay was
2467    /// actually enqueued to the module — if the relay send itself failed, the
2468    /// module never created a binding and there is nothing to tear down.
2469    fn send_abandoned_route_bind_goodbye(
2470        &self,
2471        module_sink: &crate::FrameSink,
2472        negotiated_ver: u8,
2473        module_channel: u16,
2474        module_epoch: u32,
2475    ) {
2476        let frame = match Frame::build_with_version(
2477            negotiated_ver,
2478            FrameType::Goodbye,
2479            control_flags(),
2480            module_channel,
2481            module_epoch,
2482            0,
2483            Vec::new(),
2484        ) {
2485            Ok(frame) => frame,
2486            Err(err) => {
2487                warn!(
2488                    route_channel = module_channel,
2489                    error = %err,
2490                    "failed to build GOODBYE for abandoned route.bind"
2491                );
2492                return;
2493            }
2494        };
2495        crate::forwarding::send_module_route_goodbye(
2496            &self.counters,
2497            module_sink,
2498            frame,
2499            None,
2500            "abandoned route.bind",
2501        );
2502    }
2503
2504    fn handle_hello(
2505        &self,
2506        connection_id: ConnectionId,
2507        sink: Option<crate::FrameSink>,
2508        frame: Frame,
2509    ) -> Result<Vec<Frame>, RouterError> {
2510        debug!(
2511            connection_id = connection_id.get(),
2512            corr = frame.header.corr,
2513            "handling HELLO"
2514        );
2515        // A module connection has one identity for its entire lifetime. A second
2516        // registration would leave the old registry owner behind while replacing
2517        // its forwarding endpoint and launch nonce.
2518        if self
2519            .registry
2520            .get_module_by_connection(connection_id)
2521            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2522            .is_some()
2523        {
2524            return Ok(vec![control_error_frame(
2525                &frame,
2526                "invalid_hello",
2527                "connection is already registered as a module",
2528            )?]);
2529        }
2530        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2531            Ok(value) => value,
2532            Err(err) => {
2533                return Ok(vec![control_error_frame(
2534                    &frame,
2535                    "invalid_hello",
2536                    format!("malformed HELLO body: {err}"),
2537                )?])
2538            }
2539        };
2540        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2541            return Ok(vec![control_error_frame(
2542                &frame,
2543                "invalid_capability_grammar",
2544                err.to_string(),
2545            )?]);
2546        }
2547        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2548            return Ok(vec![control_error_frame(
2549                &frame,
2550                "invalid_manifest",
2551                err.to_string(),
2552            )?]);
2553        }
2554        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2555            return Ok(vec![control_error_frame(
2556                &frame,
2557                "invalid_event_declaration",
2558                err.to_string(),
2559            )?]);
2560        }
2561        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2562            return Ok(vec![control_error_frame(
2563                &frame,
2564                "invalid_manifest",
2565                err.to_string(),
2566            )?]);
2567        }
2568        if let Some(provenance) = hello_value
2569            .get("manifest")
2570            .and_then(|manifest| manifest.get("provenance"))
2571        {
2572            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2573                return Ok(vec![control_error_frame(
2574                    &frame,
2575                    "invalid_manifest",
2576                    format!("malformed manifest provenance: {err}"),
2577                )?]);
2578            }
2579        }
2580        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2581            let module_id = hello_value["manifest"]["module_id"]
2582                .as_str()
2583                .unwrap_or("<unknown>");
2584            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2585        }
2586        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2587            let module_id = hello_value["manifest"]["module_id"]
2588                .as_str()
2589                .unwrap_or("<unknown>");
2590            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2591        }
2592        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2593            let module_id = hello_value["manifest"]["module_id"]
2594                .as_str()
2595                .unwrap_or("<unknown>");
2596            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2597        }
2598        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2599            Ok(hello) => hello,
2600            Err(err) => {
2601                return Ok(vec![control_error_frame(
2602                    &frame,
2603                    "invalid_hello",
2604                    format!("malformed HELLO body: {err}"),
2605                )?])
2606            }
2607        };
2608
2609        if hello.protocol_ver != hello.manifest.protocol_ver {
2610            return Ok(vec![control_error_frame(
2611                &frame,
2612                "invalid_manifest",
2613                format!(
2614                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2615                    hello.protocol_ver, hello.manifest.protocol_ver
2616                ),
2617            )?]);
2618        }
2619
2620        if hello.manifest.module_id.trim().is_empty() {
2621            return Ok(vec![control_error_frame(
2622                &frame,
2623                "invalid_manifest",
2624                "manifest module_id must not be empty",
2625            )?]);
2626        }
2627
2628        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2629            Ok(negotiated_ver) => negotiated_ver,
2630            Err(message) => {
2631                return Ok(vec![control_error_frame(
2632                    &frame,
2633                    "version_unsupported",
2634                    message,
2635                )?])
2636            }
2637        };
2638
2639        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2640        // swap is open for this id, the only HELLO admitted as a second process
2641        // is the one carrying the candidate's launch nonce (the swap token), and
2642        // it registers into the candidate slot rather than being refused as a
2643        // duplicate. Run after the reserved gate, a reserved module's candidate
2644        // would be refused `reserved_module` for presenting a nonce that gate
2645        // does not know. See `SupervisorHandle::swap_hello_admission`.
2646        let swap_admission = self
2647            .supervisor
2648            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2649        if swap_admission == SwapHelloAdmission::Refused {
2650            warn!(
2651                module_id = %hello.manifest.module_id,
2652                connection_id = connection_id.get(),
2653                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2654            );
2655            return Ok(vec![control_error_frame(
2656                &frame,
2657                "swap_token_invalid",
2658                format!(
2659                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2660                    hello.manifest.module_id
2661                ),
2662            )?]);
2663        }
2664        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2665
2666        // A swap candidate proves its identity with the candidate nonce above.
2667        // Other registrations receive this catalog fact only when reserved-module
2668        // policy required and accepted a nonce; unreserved registrations require
2669        // no nonce check.
2670        let nonce_verified = if swap_candidate {
2671            true
2672        } else {
2673            match self.supervisor.reserved_hello_verification(
2674                &hello.manifest.module_id,
2675                hello.launch_nonce.as_deref(),
2676            ) {
2677                Ok(verified) => verified,
2678                Err(rejection) => {
2679                    let message = match rejection {
2680                        ReservedHelloRejection::Exact { module_id } => format!(
2681                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2682                        ),
2683                        ReservedHelloRejection::Prefix {
2684                            prefix,
2685                            owner_module_id,
2686                        } => format!(
2687                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2688                            hello.manifest.module_id
2689                        ),
2690                    };
2691                    return Ok(vec![control_error_frame(
2692                        &frame,
2693                        "reserved_module",
2694                        message,
2695                    )?]);
2696                }
2697            }
2698        };
2699
2700        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2701            &hello.manifest.module_id,
2702            hello.manifest.capabilities.as_ref(),
2703        );
2704        if let Some(refusal) = reserved_capability_refusals.first() {
2705            let capability = refusal.capability.clone();
2706            let bound_module = refusal.claimants[0].clone();
2707            log_duplicate_claim_events(reserved_capability_refusals);
2708            return Ok(vec![control_error_frame(
2709                &frame,
2710                "reserved_capability",
2711                format!(
2712                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2713                    capability, bound_module, hello.manifest.module_id
2714                ),
2715            )?]);
2716        }
2717
2718        // A connection that already opened client routes must not also register as
2719        // a module: cleanup would then release only one side and leak the other.
2720        if self
2721            .forwarding
2722            .connection_has_client_routes(connection_id)
2723            .map_err(RouterError::Forwarding)?
2724        {
2725            return Ok(vec![control_error_frame(
2726                &frame,
2727                "invalid_hello",
2728                "connection has open client routes and cannot also register as a module",
2729            )?]);
2730        }
2731
2732        // Kept for scope sync authority, which goes only to the connection that
2733        // presented the module's current launch nonce. Recorded before the
2734        // registration is attempted: a connection whose registration then fails
2735        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2736        self.hello_launch_nonces
2737            .lock()
2738            .unwrap_or_else(|poisoned| poisoned.into_inner())
2739            .record(connection_id, hello.launch_nonce.as_deref());
2740        let control_ops = effective_module_control_ops(hello.control_ops);
2741        // Built before anything is registered so an encoding failure leaves no
2742        // registry or forwarding state behind.
2743        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2744        if swap_candidate {
2745            return self.register_swap_candidate(
2746                connection_id,
2747                sink,
2748                &frame,
2749                hello.manifest,
2750                negotiated_ver,
2751                control_ops,
2752                hello_ack,
2753            );
2754        }
2755        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2756        // is refused, but `catalog.list` does not show the module and the
2757        // catalog generation does not move until it is published below, once
2758        // its forwarding endpoint exists. Listing it now would advertise a
2759        // module that `route.open` still refuses for having no forwarding
2760        // connection.
2761        let registration = match self
2762            .registry
2763            .register_unpublished_verified_with_control_ops(
2764                hello.manifest,
2765                negotiated_ver,
2766                connection_id,
2767                control_ops,
2768                nonce_verified,
2769            ) {
2770            Ok(registration) => registration,
2771            Err(RegistryError::DuplicateModuleId { module_id }) => {
2772                return Ok(vec![control_error_frame(
2773                    &frame,
2774                    "duplicate_module_id",
2775                    format!(
2776                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2777                    ),
2778                )?])
2779            }
2780            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2781                return Ok(vec![control_error_frame(
2782                    &frame,
2783                    "invalid_module_id",
2784                    err.to_string(),
2785                )?])
2786            }
2787            Err(err) => {
2788                return Ok(vec![control_error_frame(
2789                    &frame,
2790                    "registry_error",
2791                    err.to_string(),
2792                )?])
2793            }
2794        };
2795        #[cfg(test)]
2796        self.hold_between_registry_and_forwarding();
2797
2798        let reply = if let Some(sink) = sink {
2799            // The forwarding table's module store is also the daemon-to-module
2800            // control-RPC lane, so every HELLO gets a live endpoint even when the
2801            // manifest has no routable provider role. Non-routable modules still
2802            // cannot receive route.bind in production: `handle_route_open` checks
2803            // the registry manifest with `target_has_required_role` before the
2804            // only production call to `begin_route_bind_relay_for` below that
2805            // route.open path. The remaining direct relay callers are unit tests
2806            // and benchmark harnesses that construct forwarding state explicitly.
2807            //
2808            // The HELLO_ACK is queued by the forwarding table itself, before the
2809            // endpoint becomes visible, and is NOT returned as a reply. A module
2810            // reads HELLO_ACK first and exits on anything else; a reply is only
2811            // written after this handler returns, by which time a route.open on
2812            // another connection could already have queued a route.bind request
2813            // for this module ahead of it.
2814            let concurrency = manifest_concurrency(&registration.manifest);
2815            if let Err(err) = self.forwarding.register_module_connection_acked(
2816                connection_id,
2817                registration.manifest.module_id.clone(),
2818                negotiated_ver,
2819                concurrency,
2820                sink,
2821                hello_ack,
2822            ) {
2823                // Forwarding registration failed, so there is no forwarding
2824                // state to tear down. Remove the registry entry, which was never
2825                // published and so never listed or counted in the catalog
2826                // generation, and signal the release watch directly.
2827                if matches!(
2828                    self.deregister_connection(
2829                        connection_id,
2830                        RegistrationEndReason::RegistrationFailed,
2831                    ),
2832                    Ok(r) if !r.is_empty()
2833                ) {
2834                    crate::supervise::notify_registration_release();
2835                }
2836                return Ok(vec![control_error_frame(
2837                    &frame,
2838                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2839                        "invalid_hello"
2840                    } else {
2841                        forwarding_error_code(&err)
2842                    },
2843                    err.to_string(),
2844                )?]);
2845            }
2846            Vec::new()
2847        } else {
2848            // No sink means no forwarding endpoint, so nothing can be routed
2849            // ahead of the ack; it goes out as the reply.
2850            vec![hello_ack]
2851        };
2852
2853        // The module is routable now (or, without a sink, as routable as it
2854        // will ever be), so the catalog may list it. Publishing bumps the
2855        // catalog generation in the same registry critical section that makes
2856        // the module listed.
2857        if !self
2858            .registry
2859            .publish_registration(connection_id)
2860            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2861        {
2862            debug!(
2863                module_id = %registration.manifest.module_id,
2864                connection_id = connection_id.get(),
2865                "registration was removed before it could be published"
2866            );
2867        }
2868
2869        // Exposure over assumption: Concurrency's serde default is pinned to the
2870        // pre-field behavior (ModuleManaged), so a management surface that is
2871        // genuinely Serial and just never declared it inherits concurrent
2872        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2873        // turns "no module has been bitten yet" into the checkable claim "no
2874        // module is exposed" -- one read of the boot log instead of a fleet
2875        // audit. Detected from the raw HELLO bytes because the serde default
2876        // deliberately erases the absent/declared distinction from the type.
2877        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2878            info!(
2879                module_id = %registration.manifest.module_id,
2880                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2881            );
2882        }
2883
2884        self.apply_registration_capabilities(&registration);
2885
2886        self.supervisor
2887            .record_activity_registration(&registration.manifest.module_id);
2888
2889        info!(
2890            module_id = %registration.manifest.module_id,
2891            module_version = %registration.manifest.module_version,
2892            negotiated_ver,
2893            routable_provider = manifest_provides_routable_role(&registration.manifest),
2894            connection_id = connection_id.get(),
2895            "module registered"
2896        );
2897
2898        Ok(reply)
2899    }
2900
2901    /// Register a HELLO the swap gate admitted into the candidate slot of the
2902    /// registry and of forwarding, where it is reachable over its own
2903    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2904    /// nothing routes to it until the supervisor cuts over.
2905    ///
2906    /// Registry first (unpublished), then forwarding, then publication, the
2907    /// same order as an ordinary HELLO; a forwarding failure removes the
2908    /// registry entry again. The swap supervisor treats a published, ready
2909    /// candidate as registered and promotes it, and promotion needs the
2910    /// candidate's forwarding endpoint, so the candidate is published only
2911    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2912    /// recompute capability verdicts or run the capability deny census
2913    /// (`enforce_capability_denies`): both are about the module in the active
2914    /// slot, which `route.open` can target. A candidate sits in the candidate
2915    /// slot, so it neither provides capabilities to other modules nor can be
2916    /// routed to until promotion moves it into the active slot.
2917    #[allow(clippy::too_many_arguments)]
2918    fn register_swap_candidate(
2919        &self,
2920        connection_id: ConnectionId,
2921        sink: Option<crate::FrameSink>,
2922        frame: &Frame,
2923        manifest: ModuleManifest,
2924        negotiated_ver: u8,
2925        control_ops: Vec<String>,
2926        hello_ack: Frame,
2927    ) -> Result<Vec<Frame>, RouterError> {
2928        let module_id = manifest.module_id.clone();
2929        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2930            manifest,
2931            negotiated_ver,
2932            connection_id,
2933            control_ops,
2934        ) {
2935            Ok(registration) => registration,
2936            Err(RegistryError::DuplicateModuleId { module_id }) => {
2937                return Ok(vec![control_error_frame(
2938                    frame,
2939                    "duplicate_module_id",
2940                    format!(
2941                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2942                    ),
2943                )?])
2944            }
2945            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2946                return Ok(vec![control_error_frame(
2947                    frame,
2948                    "invalid_module_id",
2949                    err.to_string(),
2950                )?])
2951            }
2952            Err(err) => {
2953                return Ok(vec![control_error_frame(
2954                    frame,
2955                    "registry_error",
2956                    err.to_string(),
2957                )?])
2958            }
2959        };
2960        #[cfg(test)]
2961        self.hold_between_registry_and_forwarding();
2962        let reply = if let Some(sink) = sink {
2963            // Same ordering as an ordinary HELLO: the forwarding table queues
2964            // the HELLO_ACK before the candidate endpoint is inserted, because
2965            // a module exits if its first frame after HELLO is anything else.
2966            let concurrency = manifest_concurrency(&registration.manifest);
2967            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2968                connection_id,
2969                module_id.clone(),
2970                negotiated_ver,
2971                concurrency,
2972                sink,
2973                hello_ack,
2974            ) {
2975                if matches!(
2976                    self.deregister_connection(
2977                        connection_id,
2978                        RegistrationEndReason::RegistrationFailed,
2979                    ),
2980                    Ok(r) if !r.is_empty()
2981                ) {
2982                    crate::supervise::notify_registration_release();
2983                }
2984                return Ok(vec![control_error_frame(
2985                    frame,
2986                    forwarding_error_code(&err),
2987                    err.to_string(),
2988                )?]);
2989            }
2990            Vec::new()
2991        } else {
2992            vec![hello_ack]
2993        };
2994        if !self
2995            .registry
2996            .publish_registration(connection_id)
2997            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998        {
2999            debug!(
3000                module_id = %module_id,
3001                connection_id = connection_id.get(),
3002                "swap candidate registration was removed before it could be published"
3003            );
3004        }
3005        self.supervisor.mark_swap_candidate_admitted(&module_id);
3006        info!(
3007            module_id = %module_id,
3008            module_version = %registration.manifest.module_version,
3009            negotiated_ver,
3010            ready = registration.ready,
3011            connection_id = connection_id.get(),
3012            "swap candidate registered; not routable until cutover"
3013        );
3014        Ok(reply)
3015    }
3016
3017    fn build_hello_ack(
3018        &self,
3019        frame: &Frame,
3020        negotiated_ver: u8,
3021        module_id: &str,
3022    ) -> Result<Frame, RouterError> {
3023        let ack = ModuleHelloAckBody {
3024            negotiated_ver,
3025            subc_ops: module_subc_ops(),
3026            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3027            storage: self
3028                .storage_config
3029                .as_ref()
3030                .map(|cfg| cfg.descriptor_for(module_id)),
3031            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3032        };
3033        let body = serde_json::to_vec(&ack).map_err(|err| {
3034            RouterError::backend(
3035                0,
3036                frame.header.corr,
3037                format!("failed to encode HELLO_ACK: {err}"),
3038            )
3039        })?;
3040
3041        Frame::build_with_version(
3042            negotiated_ver,
3043            FrameType::HelloAck,
3044            control_flags(),
3045            0,
3046            0,
3047            frame.header.corr,
3048            body,
3049        )
3050        .map_err(RouterError::FrameBuild)
3051    }
3052
3053    async fn handle_client_control_request(
3054        &self,
3055        ctx: &RouteCtx,
3056        frame: Frame,
3057        request: ClientControlRequest,
3058        route_open_timings: &mut Option<RouteOpenTimings>,
3059    ) -> Result<Vec<Frame>, RouterError> {
3060        if let Some(operation) = privileged_operation(&request) {
3061            let caller = self.resolve_caller(ctx.connection_id).await;
3062            let op = client_control_request_op(&request);
3063            let text = prompt_text(operation, &caller);
3064            match self.privileged_operator_prompt {
3065                PrivilegedOperatorPrompt::Log => {
3066                    info!(target: "subc_daemon::privileged_prompt", operation = op,
3067                        target = ?operation, caller = %caller.description(), summary = %text,
3068                        "privileged request would prompt the operator");
3069                }
3070                PrivilegedOperatorPrompt::Enforce => {
3071                    let key =
3072                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3073                    let stamp = self
3074                        .operator_apps
3075                        .lock()
3076                        .unwrap_or_else(|p| p.into_inner())
3077                        .get(&ctx.connection_id)
3078                        .cloned();
3079                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3080                        &self.operator_app_policy,
3081                        &self.launch_table,
3082                        stamp.as_ref(),
3083                    )
3084                    .then(|| self.shortcut_context(stamp, None, None));
3085                    let outcome = self
3086                        .forwarding
3087                        .operator_confirms()
3088                        .approve_privileged_with_shortcut(
3089                            ctx,
3090                            &frame,
3091                            op,
3092                            text,
3093                            caller.description(),
3094                            key,
3095                            shortcut,
3096                        )
3097                        .await;
3098                    if outcome.code != "confirmed" {
3099                        return Ok(vec![outcome.frame(&frame)]);
3100                    }
3101                }
3102            }
3103        }
3104        match request {
3105            ClientControlRequest::DaemonShutdownIntent { intent } => {
3106                if intent.as_str() != Some("restart") {
3107                    return Ok(vec![control_error_frame(
3108                        &frame,
3109                        "invalid_params",
3110                        "intent must be 'restart'",
3111                    )?]);
3112                }
3113                let response = ClientControlResponse::DaemonShutdownIntent {
3114                    intent: "restart".to_owned(),
3115                    expires_at_ms: self.supervisor.announce_daemon_restart(),
3116                };
3117                Ok(vec![control_response_body_frame(
3118                    &frame,
3119                    &response,
3120                    "ClientControlResponse::DaemonShutdownIntent",
3121                )?])
3122            }
3123            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3124            ClientControlRequest::CatalogList { module_id } => {
3125                self.handle_catalog_list(frame, module_id)
3126            }
3127            ClientControlRequest::RouteOpen {
3128                target,
3129                identity,
3130                consumer_identity,
3131                consumer_capabilities,
3132                role_versions,
3133                busy_refusals,
3134                admission_facts,
3135                scope,
3136                ..
3137            } => {
3138                self.handle_route_open(
3139                    ctx,
3140                    frame,
3141                    RouteOpenRequest {
3142                        target,
3143                        identity,
3144                        consumer_identity,
3145                        consumer_capabilities,
3146                        role_versions,
3147                        busy_refusals,
3148                        admission_facts,
3149                        scope,
3150                    },
3151                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3152                )
3153                .await
3154            }
3155            ClientControlRequest::RoutePoll {
3156                route_channel,
3157                route_epoch,
3158                kind,
3159            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3160            ClientControlRequest::SupervisorList {} => {
3161                // Filesystem-backed diagnostics must not hold up this connection's reader.
3162                let handler = self.clone();
3163                let sink = ctx.egress.clone();
3164                tokio::spawn(
3165                    async move {
3166                        let result = handler.handle_supervisor_list(frame.clone()).await;
3167                        let replies = match result {
3168                            Ok(replies) => replies,
3169                            Err(error) => {
3170                                control_error_frame(&frame, "backend_error", error.to_string())
3171                                    .into_iter()
3172                                    .collect()
3173                            }
3174                        };
3175                        for reply in replies {
3176                            if sink.send(reply).await.is_err() {
3177                                break;
3178                            }
3179                        }
3180                    }
3181                    .with_current_subscriber(),
3182                );
3183                Ok(Vec::new())
3184            }
3185            ClientControlRequest::SupervisorSpawnSnapshot {} => {
3186                self.handle_supervisor_spawn_snapshot(frame)
3187            }
3188            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3189                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3190            }
3191            ClientControlRequest::SupervisorRestart {
3192                module_id,
3193                drain_timeout_ms,
3194            } => {
3195                self.log_supervisor_request_received(
3196                    ctx,
3197                    frame.header.corr,
3198                    ops::SUPERVISOR_RESTART,
3199                    Some(&module_id),
3200                    None,
3201                )?;
3202                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3203                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3204                    .await
3205            }
3206            ClientControlRequest::SupervisorSwap {
3207                module_id,
3208                ready_timeout_ms,
3209            } => {
3210                self.log_supervisor_request_received(
3211                    ctx,
3212                    frame.header.corr,
3213                    ops::SUPERVISOR_SWAP,
3214                    Some(&module_id),
3215                    None,
3216                )?;
3217                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3218                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3219                    .await
3220            }
3221            ClientControlRequest::SupervisorReload { module_id } => {
3222                self.log_supervisor_request_received(
3223                    ctx,
3224                    frame.header.corr,
3225                    ops::SUPERVISOR_RELOAD,
3226                    Some(&module_id),
3227                    None,
3228                )?;
3229                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3230                self.handle_supervisor_reload(frame, module_id, requester)
3231                    .await
3232            }
3233            ClientControlRequest::SupervisorRescan { preview } => {
3234                if !preview {
3235                    self.log_supervisor_request_received(
3236                        ctx,
3237                        frame.header.corr,
3238                        ops::SUPERVISOR_RESCAN,
3239                        None,
3240                        None,
3241                    )?;
3242                }
3243                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3244                let operation = self.handle_supervisor_rescan(
3245                    frame,
3246                    if preview {
3247                        RescanMode::Preview
3248                    } else {
3249                        RescanMode::Execute
3250                    },
3251                    requester,
3252                );
3253                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3254                    .await
3255            }
3256            ClientControlRequest::SupervisorRescanChecked {
3257                preview,
3258                candidate,
3259                expect,
3260            } => {
3261                self.observe_control_wait(
3262                    ctx,
3263                    None,
3264                    ops::SUPERVISOR_RESCAN,
3265                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3266                )
3267                .await
3268            }
3269            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3270                self.log_supervisor_request_received(
3271                    ctx,
3272                    frame.header.corr,
3273                    ops::SUPERVISOR_RELEASE_RESERVED,
3274                    Some(&module_id),
3275                    None,
3276                )?;
3277                self.handle_supervisor_release_reserved(frame, module_id)
3278                    .await
3279            }
3280            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3281                self.log_supervisor_request_received(
3282                    ctx,
3283                    frame.header.corr,
3284                    ops::SUPERVISOR_SET_ENABLED,
3285                    Some(&module_id),
3286                    Some(enabled),
3287                )?;
3288                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3289                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3290                    .await
3291            }
3292            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3293                self.handle_supervisor_health_probe(ctx, frame, module_id)
3294                    .await
3295            }
3296            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3297            ClientControlRequest::SupervisorRoutes { module_id } => {
3298                self.handle_supervisor_routes(frame, module_id)
3299            }
3300            ClientControlRequest::SupervisorProvenance { module_id } => {
3301                self.handle_supervisor_provenance(ctx, frame, module_id)
3302                    .await
3303            }
3304            ClientControlRequest::SupervisorStderrTail {
3305                module_id,
3306                max_lines,
3307                max_bytes,
3308            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3309            ClientControlRequest::SupervisorTerminals { module_id } => {
3310                self.handle_supervisor_terminals(ctx, frame, module_id)
3311                    .await
3312            }
3313        }
3314    }
3315
3316    fn log_supervisor_request_received(
3317        &self,
3318        ctx: &RouteCtx,
3319        corr: u64,
3320        op: &'static str,
3321        module_id: Option<&str>,
3322        enabled: Option<bool>,
3323    ) -> Result<(), RouterError> {
3324        let caller = self.supervisor_requester(ctx, corr)?;
3325        let caller = principal_label(&caller);
3326        // `Option` fields are recorded only when present, so a request with no
3327        // module id (rescan) or no enabled flag simply omits that field.
3328        info!(
3329            target: "control",
3330            op,
3331            module_id,
3332            enabled,
3333            connection_id = ctx.connection_id.get(),
3334            caller = %caller,
3335            "supervisor request received"
3336        );
3337        Ok(())
3338    }
3339
3340    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3341        Ok(self
3342            .registry
3343            .get_module_by_connection(ctx.connection_id)
3344            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3345            .map(|registration| Principal::Reserved {
3346                module_id: registration.manifest.module_id,
3347            })
3348            .unwrap_or(Principal::Direct))
3349    }
3350
3351    fn handle_module_control_request(
3352        &self,
3353        connection_id: ConnectionId,
3354        frame: Frame,
3355        request: ModuleControlRequestFromModule,
3356    ) -> Result<Vec<Frame>, RouterError> {
3357        match request {
3358            ModuleControlRequestFromModule::CatalogUpdate {
3359                provides,
3360                capabilities,
3361                ready,
3362            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3363            ModuleControlRequestFromModule::LiveRoots {} => {
3364                let registered = self
3365                    .registry
3366                    .get_module_by_connection(connection_id)
3367                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3368                let Some(registration) = registered else {
3369                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3370                };
3371                let response = self
3372                    .forwarding
3373                    .live_roots(&registration.manifest.module_id)
3374                    .map_err(RouterError::Forwarding)?;
3375                Ok(vec![control_response_body_frame(
3376                    &frame,
3377                    &response,
3378                    "ModuleControlResponseToModule::LiveRoots",
3379                )?])
3380            }
3381            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3382                self.handle_scope_sync(connection_id, frame, generation, scopes)
3383            }
3384            ModuleControlRequestFromModule::ScopeApply {
3385                generation,
3386                upsert,
3387                end,
3388            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3389            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3390                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3391            }
3392        }
3393    }
3394
3395    fn handle_scope_served_by(
3396        &self,
3397        connection_id: ConnectionId,
3398        frame: Frame,
3399    ) -> Result<Vec<Frame>, RouterError> {
3400        let registered = self
3401            .registry
3402            .get_module_by_connection(connection_id)
3403            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3404        let Some(registered) = registered else {
3405            return Ok(vec![control_error_frame(
3406                &frame,
3407                "not_registered",
3408                "scope.served_by requires an active module registration owned by this connection",
3409            )?]);
3410        };
3411        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3412        {
3413            Ok(request) => request,
3414            Err(err) => {
3415                return Ok(vec![control_error_frame(
3416                    &frame,
3417                    "invalid_control_body",
3418                    format!("malformed scope.served_by body: {err}"),
3419                )?])
3420            }
3421        };
3422        let answer = self
3423            .scopes
3424            .read()
3425            .unwrap_or_else(|poisoned| poisoned.into_inner())
3426            .served_by(
3427                &registered.manifest.module_id,
3428                &request.scope_ref,
3429                request.scope_epoch,
3430                &request.module_id,
3431            );
3432        let reply = subc_control::ScopeServedByReply::new(
3433            answer,
3434            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3435        );
3436        Ok(vec![control_response_body_frame(
3437            &frame,
3438            &reply,
3439            "ScopeServedByReply",
3440        )?])
3441    }
3442
3443    fn handle_operator_confirm(
3444        &self,
3445        ctx: &RouteCtx,
3446        frame: Frame,
3447    ) -> Result<Vec<Frame>, RouterError> {
3448        use crate::operator_confirm::{audit, Outcome};
3449        let registration = self
3450            .registry
3451            .get_module_by_connection(ctx.connection_id)
3452            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3453        let Some(registration) = registration else {
3454            let outcome = Outcome::refusal("not_registered");
3455            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3456            return Ok(vec![outcome.frame(&frame)]);
3457        };
3458        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3459            Ok(request) => request,
3460            Err(_) => {
3461                let outcome = Outcome::refusal("invalid_control_body");
3462                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3463                return Ok(vec![outcome.frame(&frame)]);
3464            }
3465        };
3466        let declared = matches!(
3467            request.kind,
3468            Some(subc_protocol::session::OperatorConfirmKind::Operation)
3469        ) && request.operation.as_ref().is_some_and(|operation| {
3470            registration
3471                .manifest
3472                .confirm_operations
3473                .as_ref()
3474                .is_some_and(|operations| operations.contains(operation))
3475        });
3476        let module_id = registration.manifest.module_id;
3477        // Read the launch nonce (under its own lock) before taking the forwarding
3478        // table's lock below: holding forwarding while waiting on another daemon
3479        // lock risks a lock-order deadlock with paths that take them the other way.
3480        let nonce = self
3481            .hello_launch_nonces
3482            .lock()
3483            .unwrap_or_else(|p| p.into_inner())
3484            .nonce(ctx.connection_id)
3485            .map(str::to_owned);
3486        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3487            self.supervisor
3488                .spawned_consumer_authorized(&module_id, nonce)
3489        });
3490        let confirms = self.forwarding.operator_confirms();
3491        self.forwarding
3492            .with_operator_route(
3493                ctx.connection_id,
3494                request.route_channel,
3495                request.route_epoch,
3496                |binding| {
3497                    let shortcut = binding
3498                        .filter(|binding| {
3499                            nonce_proven
3500                                && crate::operator_confirm::valid_summary(&request.summary)
3501                                && declared
3502                                && crate::operator_confirm::ShortcutContext::route_eligible(
3503                                    &self.operator_app_policy,
3504                                    &self.launch_table,
3505                                    binding,
3506                                )
3507                        })
3508                        .map(|binding| {
3509                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3510                                &self.operator_app_policy,
3511                                binding,
3512                            );
3513                            let stamp = if phone.is_some() {
3514                                None
3515                            } else {
3516                                binding.operator_app.clone()
3517                            };
3518                            self.shortcut_context(
3519                                stamp,
3520                                phone,
3521                                Some((request.route_channel, request.route_epoch)),
3522                            )
3523                        });
3524                    confirms.admit_with_shortcut(
3525                        ctx,
3526                        frame,
3527                        module_id,
3528                        nonce_proven,
3529                        request,
3530                        binding,
3531                        shortcut,
3532                    )
3533                },
3534            )
3535            .map_err(RouterError::Forwarding)
3536    }
3537
3538    fn shortcut_context(
3539        &self,
3540        stamp: Option<crate::operator_apps::OperatorAppStamp>,
3541        phone_fingerprint: Option<String>,
3542        route: Option<(u16, u32)>,
3543    ) -> crate::operator_confirm::ShortcutContext {
3544        crate::operator_confirm::ShortcutContext {
3545            ledger: self.activity.clone(),
3546            forwarding: self.forwarding.clone(),
3547            launches: self.launch_table.clone(),
3548            stamp,
3549            phone_fingerprint,
3550            route,
3551            connections: self.operator_apps.clone(),
3552            #[cfg(test)]
3553            hooks: Default::default(),
3554        }
3555    }
3556
3557    /// Explicit test-only phone policy; daemon configuration cannot select it.
3558    #[cfg(any(test, feature = "test-support"))]
3559    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3560        self.operator_app_policy.phone_enabled = enabled;
3561        self
3562    }
3563
3564    /// Resolve folders before taking the scope-table lock: a slow filesystem
3565    /// must not hold up admissions for unrelated scoped routes.
3566    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3567        let mut scopes = ScopeRecords::from(scopes);
3568        for (index, record) in scopes.records.iter_mut().enumerate() {
3569            if let Some(refusal) = Self::canonicalize_workdir(record) {
3570                scopes.refusals.insert(index, refusal);
3571            }
3572        }
3573        scopes
3574    }
3575
3576    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3577        let workdir = record.workdir.as_mut()?;
3578        let path = std::path::Path::new(workdir);
3579        if !path.is_absolute() {
3580            return Some((
3581                error_codes::INVALID_WORKDIR,
3582                "workdir must be absolute".to_string(),
3583            ));
3584        }
3585        match ProjectRootId::from_path_allowing_missing(path) {
3586            Ok(root) => match root.as_path().to_str() {
3587                Some(canonical) => {
3588                    *workdir = canonical.to_string();
3589                    None
3590                }
3591                None => Some((
3592                    error_codes::INVALID_WORKDIR,
3593                    "canonical workdir is not UTF-8".to_string(),
3594                )),
3595            },
3596            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3597        }
3598    }
3599
3600    /// `scope.sync`: the owner is the module registered on this connection.
3601    /// A connection with no registration (every client connection, `direct`
3602    /// included) is refused `not_registered` before the table is consulted.
3603    fn handle_scope_sync(
3604        &self,
3605        connection_id: ConnectionId,
3606        frame: Frame,
3607        generation: u64,
3608        scopes: Vec<ScopeRecord>,
3609    ) -> Result<Vec<Frame>, RouterError> {
3610        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3611    }
3612
3613    fn handle_scope_change(
3614        &self,
3615        connection_id: ConnectionId,
3616        frame: Frame,
3617        generation: u64,
3618        scopes: Vec<ScopeRecord>,
3619        end: Option<Vec<ScopeEnd>>,
3620    ) -> Result<Vec<Frame>, RouterError> {
3621        let is_apply = end.is_some();
3622        let Some(registration) = self
3623            .registry
3624            .get_module_by_connection(connection_id)
3625            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3626        else {
3627            return Ok(vec![control_error_frame(
3628                &frame,
3629                "not_registered",
3630                "scope.sync and scope.apply require an active module registration owned by this connection",
3631            )?]);
3632        };
3633        let owner = registration.manifest.module_id;
3634        let scopes = Self::prepare_scope_records(scopes);
3635        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3636        let is_current_launch = |connection: ConnectionId| {
3637            self.hello_launch_nonces
3638                .lock()
3639                .unwrap_or_else(|poisoned| poisoned.into_inner())
3640                .presented(connection, current_nonce.as_deref())
3641        };
3642        // Lock order is the scope table, then the forwarding table: the new
3643        // tags are published, and the routes the change closes are selected,
3644        // while the scope table is still write-locked, so no admission can read
3645        // a record whose tag is not yet published.
3646        let mut table = self
3647            .scopes
3648            .write()
3649            .unwrap_or_else(|poisoned| poisoned.into_inner());
3650        let outcome = match end {
3651            Some(end) => table.apply(
3652                &owner,
3653                connection_id,
3654                is_current_launch,
3655                generation,
3656                scopes,
3657                end,
3658            ),
3659            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3660        };
3661        let drained = match &outcome {
3662            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3663            Err(_) => Vec::new(),
3664        };
3665        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3666        drop(table);
3667        if has_deadlines {
3668            self.scope_deadline_added.notify_one();
3669        }
3670        match outcome {
3671            Ok(applied) => {
3672                let counts = ScopeOutcomeCounts::of(&applied.results);
3673                info!(
3674                    owner = %owner,
3675                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3676                    generation,
3677                    records = applied.results.len(),
3678                    created = counts.created,
3679                    replaced = counts.replaced,
3680                    updated = counts.updated,
3681                    unchanged = counts.unchanged,
3682                    refused = counts.refused,
3683                    ended = applied.ended.len(),
3684                    tag_changes = applied.tag_changes.len(),
3685                    routes_closed = drained.len(),
3686                    "scope change accepted"
3687                );
3688                // An accepted scope change can still refuse individual records, and the
3689                // owner is the only party that sees the reply. Name them here so
3690                // an operator can tell a refused session from a missing one
3691                // without the owner's logs. Capped so a call that refuses
3692                // thousands cannot flood the log; the count above is complete.
3693                for refused in applied
3694                    .results
3695                    .iter()
3696                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3697                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3698                {
3699                    warn!(
3700                        owner = %owner,
3701                        generation,
3702                        scope_ref = %refused.scope_ref,
3703                        scope_epoch = refused.scope_epoch,
3704                        code = refused.code.as_deref().unwrap_or(""),
3705                        "scope record refused"
3706                    );
3707                }
3708                self.close_scope_drained_routes(drained);
3709                let response = if is_apply {
3710                    ModuleControlResponseToModule::ScopeApply {
3711                        generation,
3712                        results: applied.results,
3713                        end_results: applied.end_results,
3714                        ended: applied.ended,
3715                    }
3716                } else {
3717                    ModuleControlResponseToModule::ScopeSync {
3718                        generation,
3719                        results: applied.results,
3720                        ended: applied.ended,
3721                    }
3722                };
3723                Ok(vec![control_response_body_frame(
3724                    &frame,
3725                    &response,
3726                    "scope change response",
3727                )?])
3728            }
3729            Err(refusal) => {
3730                info!(
3731                    owner = %owner,
3732                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3733                    generation,
3734                    code = refusal.code,
3735                    "scope change refused"
3736                );
3737                Ok(vec![control_error_frame(
3738                    &frame,
3739                    refusal.code,
3740                    refusal.message,
3741                )?])
3742            }
3743        }
3744    }
3745
3746    /// Tell both ends of each route a scope change closed. The module gets a
3747    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3748    /// the client's route handle. The client also gets `route.closed` with the
3749    /// scope reason, one push per module and reason, so it can tell a revoked
3750    /// route from an ordinary close and not reopen it.
3751    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3752        if drained.is_empty() {
3753            return;
3754        }
3755        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3756            BTreeMap::new();
3757        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3758        for route in drained {
3759            warn!(
3760                module_id = %route.module_id,
3761                reason = ?route.reason,
3762                client_connection_id = route.client.connection_id.get(),
3763                route_channel = route.client.channel,
3764                "closing route because its scope changed"
3765            );
3766            pushes
3767                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3768                .or_insert_with(|| (route.reason, Vec::new()))
3769                .1
3770                .push(EndpointRoute {
3771                    goodbye_target: route.client.clone(),
3772                    principal: Principal::Unverified,
3773                    bound_at: Instant::now(),
3774                    draining: false,
3775                    drain_reason: None,
3776                });
3777            goodbyes.push(route.module);
3778            goodbyes.push(route.client);
3779        }
3780        for ((module_id, _), (reason, routes)) in pushes {
3781            send_route_control_pushes(
3782                &self.forwarding,
3783                routes,
3784                ClientControlPush::RouteClosed {
3785                    module_id,
3786                    channels: Vec::new(),
3787                    reason,
3788                    drained: false,
3789                    abandoned: 0,
3790                    excluded_subscriptions: 0,
3791                    terminal: Some(false),
3792                },
3793            );
3794        }
3795        self.emit_route_goodbyes(goodbyes);
3796    }
3797
3798    /// `scope.describe`: any registered module may read any scope, because a
3799    /// provider must read the scope a route it serves is stamped with.
3800    fn handle_scope_describe(
3801        &self,
3802        connection_id: ConnectionId,
3803        frame: Frame,
3804        owner: Principal,
3805        scope_ref: String,
3806    ) -> Result<Vec<Frame>, RouterError> {
3807        let registered = self
3808            .registry
3809            .get_module_by_connection(connection_id)
3810            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3811        let Some(registered) = registered else {
3812            return Ok(vec![control_error_frame(
3813                &frame,
3814                "not_registered",
3815                "scope.describe requires an active module registration owned by this connection",
3816            )?]);
3817        };
3818        let mut description = self
3819            .scopes
3820            .read()
3821            .unwrap_or_else(|poisoned| poisoned.into_inner())
3822            .describe(&owner, &scope_ref);
3823        // Describe readers also decode strict ScopeAttributes. Attribution is
3824        // safe to omit unless the requesting module declares it understands it.
3825        if !registered
3826            .manifest
3827            .capabilities
3828            .as_ref()
3829            .is_some_and(|caps| {
3830                caps.provides
3831                    .iter()
3832                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3833            })
3834        {
3835            if let Some(stamp) = description.stamp.as_mut() {
3836                stamp.attributes.on_behalf_of_agent = None;
3837            }
3838        }
3839        if !registered
3840            .manifest
3841            .capabilities
3842            .as_ref()
3843            .is_some_and(|caps| {
3844                caps.provides
3845                    .iter()
3846                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3847            })
3848        {
3849            if let Some(stamp) = description.stamp.as_mut() {
3850                stamp.attributes.flow_version = None;
3851                stamp.attributes.flow_code_hash = None;
3852                stamp.attributes.flow_install_id = None;
3853            }
3854        }
3855        let owner_configured = match &owner {
3856            // Ask whether the owner is configured (`is_configured`), not
3857            // whether it is on the roster (`get(..).is_some()`): a supervised
3858            // module's process can register and describe a scope before the
3859            // supervisor has put it on the roster.
3860            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3861            _ => false,
3862        };
3863        let response = ModuleControlResponseToModule::ScopeDescribe {
3864            status: description.status,
3865            scope_epoch: description.scope_epoch,
3866            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3867            owner_synced: description.owner_synced,
3868            owner_configured,
3869            scope: description.stamp,
3870        };
3871        Ok(vec![control_response_body_frame(
3872            &frame,
3873            &response,
3874            "ModuleControlResponseToModule::ScopeDescribe",
3875        )?])
3876    }
3877
3878    fn handle_catalog_update(
3879        &self,
3880        connection_id: ConnectionId,
3881        frame: Frame,
3882        provides: Vec<ProviderRole>,
3883        capabilities: Option<CapabilityDeclarations>,
3884        ready: Option<bool>,
3885    ) -> Result<Vec<Frame>, RouterError> {
3886        self.refresh_capability_requirements();
3887        let Some(registration) = self
3888            .registry
3889            .get_module_by_connection(connection_id)
3890            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3891        else {
3892            return Ok(vec![control_error_frame(
3893                &frame,
3894                "not_registered",
3895                "catalog.update requires an active module registration owned by this connection",
3896            )?]);
3897        };
3898
3899        if let Some(message) =
3900            catalog_update_frozen_field_message(&registration.manifest, &provides)
3901        {
3902            return Ok(vec![control_error_frame(
3903                &frame,
3904                "catalog_update_frozen_field",
3905                message,
3906            )?]);
3907        }
3908
3909        let mut candidate = registration.manifest.clone();
3910        candidate.provides = provides.clone();
3911        candidate.capabilities = capabilities
3912            .clone()
3913            .or_else(|| registration.manifest.capabilities.clone());
3914        if let Some(rich_ui) = &candidate.rich_ui {
3915            if let Err(err) = rich_ui.validate() {
3916                return Ok(vec![control_error_frame(
3917                    &frame,
3918                    "invalid_manifest",
3919                    err.to_string(),
3920                )?]);
3921            }
3922        }
3923        if let Err(err) = candidate.validate_capability_grammar() {
3924            return Ok(vec![control_error_frame(
3925                &frame,
3926                "invalid_capability_grammar",
3927                err.to_string(),
3928            )?]);
3929        }
3930
3931        // Updates must honor the same reserved owner as initial registration;
3932        // otherwise an empty HELLO could acquire the claim after admission.
3933        let mut conflicts = self
3934            .capability_evaluator
3935            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3936        if let Some(conflict) = conflicts.first() {
3937            let message = format!(
3938                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3939                conflict.capability, conflict.claimants[0], candidate.module_id
3940            );
3941            for conflict in &mut conflicts {
3942                conflict.source = DuplicateClaimSource::CatalogUpdate;
3943            }
3944            log_duplicate_claim_events(conflicts);
3945            return Ok(vec![control_error_frame(
3946                &frame,
3947                "reserved_capability",
3948                message,
3949            )?]);
3950        }
3951
3952        let updated = self
3953            .registry
3954            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3955            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3956        if updated.is_none() {
3957            return Ok(vec![control_error_frame(
3958                &frame,
3959                "not_registered",
3960                "catalog.update requires an active module registration owned by this connection",
3961            )?]);
3962        }
3963        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3964            log_duplicate_claim_events(
3965                self.capability_evaluator
3966                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3967            );
3968        }
3969        if capability_census_trigger(
3970            registration.manifest.capabilities.as_ref(),
3971            updated
3972                .as_ref()
3973                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3974        ) {
3975            self.enforce_capability_denies();
3976        }
3977        self.refresh_capability_requirements();
3978
3979        let response = ModuleControlResponseToModule::CatalogUpdate {};
3980        control_response_body_frame(
3981            &frame,
3982            &response,
3983            "ModuleControlResponseToModule::CatalogUpdate",
3984        )
3985        .map(|frame| vec![frame])
3986    }
3987
3988    pub(crate) fn with_activity_ledger_config(
3989        mut self,
3990        run_dir: &Path,
3991        incarnation: String,
3992        config: crate::activity::ActivityConfig,
3993    ) -> Self {
3994        self.activity = Some(crate::activity::ActivityLedger::open_config(
3995            run_dir.join("activity.jsonl"),
3996            incarnation,
3997            config,
3998        ));
3999        self.supervisor.set_activity(self.activity.clone());
4000        self
4001    }
4002
4003    pub(crate) async fn configure_activity(
4004        &self,
4005        config: crate::activity::ActivityConfig,
4006    ) -> Result<(), &'static str> {
4007        match &self.activity {
4008            Some(ledger) => ledger.configure(config).await,
4009            None => Ok(()),
4010        }
4011    }
4012
4013    fn cancel_activity_subscription(
4014        &self,
4015        ctx: &RouteCtx,
4016        frame: &Frame,
4017    ) -> Result<bool, RouterError> {
4018        let task = self
4019            .activity_streams
4020            .lock()
4021            .unwrap_or_else(|p| p.into_inner())
4022            .remove(&(ctx.connection_id, frame.header.corr));
4023        let Some(task) = task else {
4024            return Ok(false);
4025        };
4026        task.abort();
4027        let end = Frame::build_with_version(
4028            response_version(frame),
4029            FrameType::StreamEnd,
4030            control_flags(),
4031            0,
4032            0,
4033            frame.header.corr,
4034            Vec::new(),
4035        )
4036        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4037        let sink = ctx.egress.clone();
4038        tokio::spawn(async move {
4039            let _ = sink.send(end).await;
4040        });
4041        Ok(true)
4042    }
4043
4044    async fn handle_activity_read(
4045        &self,
4046        ctx: &RouteCtx,
4047        frame: Frame,
4048    ) -> Result<Vec<Frame>, RouterError> {
4049        use subc_control::{
4050            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4051            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4052        };
4053        #[derive(Deserialize)]
4054        struct Identity {
4055            #[serde(default)]
4056            consumer_identity: Option<ConsumerIdentity>,
4057        }
4058        // A registered module connection already identifies its caller. Ignore
4059        // consumer_identity there so a module cannot borrow a mirror's identity.
4060        // Only separate consumer connections use the Direct/nonce trust boundary.
4061        let principal = match self
4062            .forwarding
4063            .module_id_for_connection(ctx.connection_id)
4064            .map_err(RouterError::Forwarding)?
4065        {
4066            Some(module_id) => Ok(Principal::Reserved { module_id }),
4067            None => {
4068                let identity: Identity = match serde_json::from_slice(&frame.body) {
4069                    Ok(identity) => identity,
4070                    Err(_) => {
4071                        return Ok(vec![control_error_frame(
4072                            &frame,
4073                            ACTIVITY_MALFORMED,
4074                            "malformed activity read",
4075                        )?])
4076                    }
4077                };
4078                self.route_open_principal(&frame, identity.consumer_identity)?
4079            }
4080        };
4081        let admitted = matches!(principal, Ok(Principal::Direct))
4082            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4083        if !admitted {
4084            return Ok(vec![control_error_frame(
4085                &frame,
4086                ACTIVITY_READ_FORBIDDEN,
4087                "activity reads require an operator or trusted mirror principal",
4088            )?]);
4089        }
4090        let Some(ledger) = &self.activity else {
4091            return Ok(vec![control_error_frame(
4092                &frame,
4093                ACTIVITY_UNAVAILABLE,
4094                "activity ledger unavailable",
4095            )?]);
4096        };
4097        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4098            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4099        {
4100            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4101                Ok(request) => ledger.query(request).await,
4102                Err(_) => Err(ACTIVITY_MALFORMED),
4103            };
4104            return match result {
4105                Ok(reply) => Ok(vec![control_response_body_frame(
4106                    &frame,
4107                    &reply,
4108                    ops::ACTIVITY_QUERY,
4109                )?]),
4110                Err(code) => Ok(vec![control_error_frame(
4111                    &frame,
4112                    code,
4113                    "activity query refused",
4114                )?]),
4115            };
4116        }
4117        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4118            Ok(request) => request,
4119            Err(_) => {
4120                return Ok(vec![control_error_frame(
4121                    &frame,
4122                    ACTIVITY_MALFORMED,
4123                    "malformed activity subscription",
4124                )?])
4125            }
4126        };
4127        let mut subscription = match ledger.subscribe(request.since).await {
4128            Ok(subscription) => subscription,
4129            Err(code) => {
4130                return Ok(vec![control_error_frame(
4131                    &frame,
4132                    code,
4133                    "activity subscription refused",
4134                )?])
4135            }
4136        };
4137        let key = (ctx.connection_id, frame.header.corr);
4138        let sink = ctx.egress.clone();
4139        let mut streams = self
4140            .activity_streams
4141            .lock()
4142            .unwrap_or_else(|p| p.into_inner());
4143        streams.retain(|_, task| !task.is_finished());
4144        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4145            return Ok(vec![control_error_frame(
4146                &frame,
4147                "duplicate_subscription",
4148                "activity subscription correlation id already in use",
4149            )?]);
4150        }
4151        let task = tokio::spawn(async move {
4152            loop {
4153                let entry = if let Some(entry) = subscription.replay.pop_front() {
4154                    Some(entry)
4155                } else {
4156                    subscription.receiver.recv().await
4157                };
4158                let Some(entry) = entry else {
4159                    break;
4160                };
4161                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4162                    return;
4163                };
4164                let Ok(next) = Frame::build_with_version(
4165                    response_version(&frame),
4166                    FrameType::StreamData,
4167                    control_flags(),
4168                    0,
4169                    0,
4170                    frame.header.corr,
4171                    body,
4172                ) else {
4173                    return;
4174                };
4175                if sink.send(next).await.is_err() {
4176                    return;
4177                }
4178            }
4179            // Dropping a full subscriber's sender drains its accepted entries
4180            // before the terminal error, preserving their order on the wire.
4181            if let Some(code) = subscription.terminal_code() {
4182                if let Ok(end) =
4183                    control_error_frame(&frame, code, "activity subscriber fell behind")
4184                {
4185                    let _ = sink.send(end).await;
4186                }
4187            }
4188        });
4189        streams.insert(key, task.abort_handle());
4190        Ok(Vec::new())
4191    }
4192
4193    async fn handle_activity_record(
4194        &self,
4195        ctx: &RouteCtx,
4196        frame: Frame,
4197    ) -> Result<Vec<Frame>, RouterError> {
4198        use subc_control::{
4199            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4200            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4201        };
4202        let result = async {
4203            let module = self
4204                .forwarding
4205                .module_id_for_connection(ctx.connection_id)
4206                .ok()
4207                .flatten()
4208                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4209            let request: ActivityRecordRequest =
4210                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4211            crate::activity::validate(&module, &request)?;
4212            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4213            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4214            let at_ms = std::time::SystemTime::now()
4215                .duration_since(std::time::UNIX_EPOCH)
4216                .unwrap_or_default()
4217                .as_millis() as u64;
4218            ledger.record(module, request, actor, at_ms).await
4219        }
4220        .await;
4221        match result {
4222            Ok(entry) => Ok(vec![control_response_body_frame(
4223                &frame,
4224                &ActivityRecordResponse { seq: entry.seq },
4225                "activity.record",
4226            )?]),
4227            Err(code) => Ok(vec![control_error_frame(
4228                &frame,
4229                code,
4230                "activity recording refused",
4231            )?]),
4232        }
4233    }
4234
4235    fn activity_actor(
4236        &self,
4237        connection: ConnectionId,
4238        module: &str,
4239        on: Option<&subc_control::ActivityOn>,
4240    ) -> Result<subc_control::ActivityActor, &'static str> {
4241        use crate::forwarding::{DataRoute, DataRouteState};
4242        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4243        let principal = |id: &str| Principal::Reserved {
4244            module_id: id.to_owned(),
4245        };
4246        let mut actor = ActivityActor {
4247            principal: principal(module),
4248            scope_owner: None,
4249            agent_id: None,
4250            run_id: None,
4251            flow_id: None,
4252            project_root: None,
4253            workdir: None,
4254            project_id: None,
4255        };
4256        let scope = match on {
4257            None => return Ok(actor),
4258            Some(ActivityOn::Scope(scope)) => {
4259                let selector = subc_protocol::scope::ScopeSelector::new(
4260                    principal(module),
4261                    scope.scope_ref.clone(),
4262                    Some(scope.epoch),
4263                );
4264                self.scopes
4265                    .read()
4266                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4267                    .attribute(&principal(module), module, &selector, true)
4268                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4269            }
4270            Some(ActivityOn::Route(route)) => {
4271                let DataRoute::Module(DataRouteState::Bound(binding)) = self
4272                    .forwarding
4273                    .lookup_data_route(connection, route.channel, route.epoch)
4274                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4275                else {
4276                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4277                };
4278                if binding.module_id != module {
4279                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4280                }
4281                actor.principal = binding.principal.clone();
4282                actor.project_root = binding
4283                    .project_root
4284                    .as_ref()
4285                    .map(|root| root.as_path().to_path_buf());
4286                let Some(bound) = &binding.scope else {
4287                    return Ok(actor);
4288                };
4289                let owner = principal(&bound.owner);
4290                let selector = subc_protocol::scope::ScopeSelector::new(
4291                    owner.clone(),
4292                    bound.scope_ref.clone(),
4293                    Some(bound.tag.scope_epoch),
4294                );
4295                let admitted = self
4296                    .scopes
4297                    .read()
4298                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4299                    .attribute(&owner, module, &selector, true)
4300                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4301                if admitted.tag != bound.tag {
4302                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4303                }
4304                admitted
4305            }
4306        };
4307        actor.scope_owner = Some(scope.stamp.owner);
4308        actor.agent_id = scope.stamp.attributes.agent_id;
4309        actor.run_id = scope.stamp.attributes.run_id;
4310        actor.flow_id = scope.stamp.attributes.flow_id;
4311        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4312        actor.project_id = scope.stamp.project_id;
4313        Ok(actor)
4314    }
4315
4316    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4317        self.refresh_capability_requirements();
4318        // A bare connection count is ambiguous between many clients holding a
4319        // route each and one client accumulating hundreds, so publish the
4320        // concentration alongside it. Route state is best-effort here: a
4321        // diagnostic endpoint must still answer if the forwarding lock is
4322        // contended.
4323        let mut counters = self.counters.snapshot();
4324        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4325            obj.insert(
4326                "activity_unparseable_lines".into(),
4327                ledger.read_errors().into(),
4328            );
4329            obj.insert(
4330                "activity_lifecycle_append_failures".into(),
4331                ledger.lifecycle_failures().into(),
4332            );
4333            obj.insert(
4334                "activity_started_digest_skips".into(),
4335                ledger.lifecycle_digest_skips().into(),
4336            );
4337        }
4338        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4339            self.forwarding.client_route_concentration(),
4340            counters.as_object_mut(),
4341        ) {
4342            obj.insert(
4343                "client_connections_with_routes".into(),
4344                connections_with_routes.into(),
4345            );
4346            obj.insert("max_routes_on_one_connection".into(), max.into());
4347        }
4348        // A module that is being fast-refused and a module that is fine look
4349        // identical from a client that retries and succeeds, so name the open
4350        // breakers here. This rides the existing free-form counters object
4351        // rather than a new wire field, so no sibling that deserializes
4352        // `ServerDescribe` has to be rebuilt to keep reading it.
4353        if let (Some(open_breakers), Some(obj)) = (
4354            self.route_bind_breakers.open_snapshot(),
4355            counters.as_object_mut(),
4356        ) {
4357            obj.insert("route_bind_breakers_open".into(), open_breakers);
4358        }
4359        // A daemon whose local endpoint could not be bound keeps serving TCP,
4360        // and the connection file names no endpoint. Name the reason here, in
4361        // the free-form counters object, so an operator reading daemon facts
4362        // sees it without searching the log, and no wire field changes.
4363        if let (Some(reason), Some(obj)) = (
4364            self.local_endpoint_unavailable.clone(),
4365            counters.as_object_mut(),
4366        ) {
4367            obj.insert("local_endpoint_unavailable".into(), reason.into());
4368        }
4369        let response = ClientControlResponse::ServerDescribe {
4370            protocol_ver: PROTOCOL_VERSION,
4371            subc_ops: subc_ops(),
4372            capabilities: self.describe_capabilities(),
4373            connected_clients: self.connected_clients.count(),
4374            counters: Some(counters),
4375            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4376            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4377            capability_requirements: self.capability_requirement_statuses(),
4378            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4379            config_path: self
4380                .rescan
4381                .as_ref()
4382                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4383        };
4384        Ok(vec![control_response_body_frame(
4385            &frame,
4386            &response,
4387            "ClientControlResponse::ServerDescribe",
4388        )?])
4389    }
4390
4391    fn handle_catalog_list(
4392        &self,
4393        frame: Frame,
4394        module_id: Option<String>,
4395    ) -> Result<Vec<Frame>, RouterError> {
4396        let (generation, modules) = self
4397            .registry
4398            .list_modules_with_nonce_verification()
4399            .map_err(|err| {
4400                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4401            })?;
4402        let entries = modules
4403            .into_iter()
4404            .filter(|(registration, _)| {
4405                module_id
4406                    .as_deref()
4407                    .map(|wanted| registration.manifest.module_id == wanted)
4408                    .unwrap_or(true)
4409            })
4410            .map(|(registration, nonce_verified)| {
4411                let not_ready = self.not_ready_reason(&registration);
4412                let roles = registration.manifest.provides;
4413                CatalogEntry::new(
4414                    registration.manifest.module_id,
4415                    roles,
4416                    registration.control_ops,
4417                )
4418                .with_ready(not_ready.is_none())
4419                .with_not_ready(not_ready)
4420                .with_module_version(Some(registration.manifest.module_version))
4421                .with_provenance(registration.manifest.provenance)
4422                .with_capabilities(registration.manifest.capabilities)
4423                .with_self_signals(registration.manifest.self_signals)
4424                .with_events(registration.manifest.events)
4425                .with_operation_delivery(registration.manifest.operation_delivery)
4426                .with_operation_payload(registration.manifest.operation_payload)
4427                .with_flow_operations(registration.manifest.flow_operations)
4428                .with_confirm_operations(registration.manifest.confirm_operations)
4429                .with_rich_ui(registration.manifest.rich_ui)
4430                .with_role_declarations(registration.manifest.role_declarations)
4431                .with_nonce_verified(nonce_verified)
4432            })
4433            .collect();
4434        let response = ClientControlResponse::CatalogList {
4435            generation,
4436            modules: entries,
4437            subc_ops: subc_ops(),
4438        };
4439        Ok(vec![control_response_body_frame(
4440            &frame,
4441            &response,
4442            "ClientControlResponse::CatalogList",
4443        )?])
4444    }
4445
4446    /// The principal a route.open's `consumer_identity` attests: `Direct`
4447    /// when it presents none, `Reserved` when its launch nonce verifies, and
4448    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4449    /// can run before any refusal check without changing which check refuses.
4450    fn attest_route_open_consumer(
4451        &self,
4452        consumer_identity: Option<&ConsumerIdentity>,
4453    ) -> Option<Principal> {
4454        let Some(consumer_identity) = consumer_identity else {
4455            return Some(Principal::Direct);
4456        };
4457        self.supervisor
4458            .spawned_consumer_authorized(
4459                &consumer_identity.module_id,
4460                &consumer_identity.launch_nonce,
4461            )
4462            .then(|| Principal::Reserved {
4463                module_id: consumer_identity.module_id.clone(),
4464            })
4465    }
4466
4467    /// The caller fields for a refusal that only has the raw request frame,
4468    /// such as the connection-level admission refusal sent before the request
4469    /// reaches [`Self::handle_route_open`].
4470    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4471        let Ok(ClientControlRequest::RouteOpen {
4472            identity,
4473            consumer_identity,
4474            scope,
4475            ..
4476        }) = parse_client_control_request(&frame.body)
4477        else {
4478            return RouteOpenCaller::unknown();
4479        };
4480        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4481        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4482    }
4483
4484    fn route_open_principal(
4485        &self,
4486        frame: &Frame,
4487        consumer_identity: Option<ConsumerIdentity>,
4488    ) -> Result<Result<Principal, Frame>, RouterError> {
4489        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4490        self.route_open_principal_attested(frame, consumer_identity, attested)
4491    }
4492
4493    /// [`Self::route_open_principal`] with `attested` already taken by
4494    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4495    /// open checks the launch nonce once.
4496    fn route_open_principal_attested(
4497        &self,
4498        frame: &Frame,
4499        consumer_identity: Option<ConsumerIdentity>,
4500        attested: Option<Principal>,
4501    ) -> Result<Result<Principal, Frame>, RouterError> {
4502        if let Some(principal) = attested {
4503            return Ok(Ok(principal));
4504        }
4505        let Some(consumer_identity) = consumer_identity else {
4506            return Ok(Ok(Principal::Direct));
4507        };
4508
4509        Ok(Err(control_error_frame(
4510            frame,
4511            "bad_consumer_identity",
4512            format!(
4513                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4514                consumer_identity.module_id
4515            ),
4516        )?))
4517    }
4518
4519    /// Ordinary `route.open` refusals go through here; admission and breaker
4520    /// refusals log separately with their capacity or breaker state. The daemon can
4521    /// attest which code it sent: without the event, a client's "the daemon
4522    /// refused me" and the daemon's own view could only be reconciled by
4523    /// argument. Malformed input (`invalid_project_root`) does not come here;
4524    /// rejecting a request that was never a valid open is not a refusal of one.
4525    #[allow(clippy::too_many_arguments)]
4526    fn route_open_refusal_frame(
4527        &self,
4528        ctx: &RouteCtx,
4529        caller: &RouteOpenCaller,
4530        frame: &Frame,
4531        module_id: &str,
4532        reason: &'static str,
4533        code: &'static str,
4534        message: impl Into<String>,
4535    ) -> Result<Frame, RouterError> {
4536        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4537        control_error_frame(frame, code, message.into())
4538    }
4539
4540    /// Answer a waiting `route.open` whose relayed bind was refused.
4541    ///
4542    /// `daemon_authored` is provenance, not a code match: the daemon
4543    /// synthesized the refusal itself (a drain, a late reply, a scope that
4544    /// ended mid-bind) or the module refused the bind with its own body. A
4545    /// daemon refusal carries the daemon-origin bit; a module's own
4546    /// rejection is relayed with the bit clear, so a module that refuses
4547    /// with `module_reloading` still cannot mint a daemon refusal.
4548    #[allow(clippy::too_many_arguments)]
4549    fn route_bind_rejection(
4550        &self,
4551        ctx: &RouteCtx,
4552        caller: &RouteOpenCaller,
4553        frame: &Frame,
4554        target_module_id: &str,
4555        body: ErrorBody,
4556        daemon_authored: bool,
4557    ) -> Result<Vec<Frame>, RouterError> {
4558        // The daemon's own commit re-check refused the bind because the
4559        // scope ended or changed after admission. The module accepted;
4560        // counting it as a module rejection would blame the module. This
4561        // branch only runs for daemon-synthesized bodies: a module that
4562        // refuses with a scope code is answered as a module rejection below.
4563        if daemon_authored {
4564            let scope_code = match body.code.as_str() {
4565                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4566                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4567                _ => None,
4568            };
4569            if let Some(code) = scope_code {
4570                self.observe_route_open_refusal(
4571                    ctx,
4572                    caller,
4573                    target_module_id,
4574                    "scope_changed_before_commit",
4575                    code,
4576                );
4577                return Ok(vec![control_error_body_frame(frame, body)?]);
4578            }
4579        }
4580        self.counters
4581            .increment_route_open_refused("module_rejected");
4582        info!(
4583            target: "control",
4584            code = "module_rejected",
4585            module_code = ?body.code,
4586            module_id = ?target_module_id,
4587            connection_id = ctx.connection_id.get(),
4588            principal = %caller.principal,
4589            scope = %caller.scope,
4590            harness = %caller.harness,
4591            "route.open refused"
4592        );
4593        let refusal = if daemon_authored {
4594            control_error_body_frame(frame, body)?
4595        } else {
4596            control_relayed_error_body_frame(frame, body)?
4597        };
4598        Ok(vec![refusal])
4599    }
4600
4601    /// Refuse a `route.open` because the target module's bind-relay breaker is
4602    /// open, without attempting the relay.
4603    ///
4604    /// The wire code is `module_timeout`, which is the truth (the module has
4605    /// not been answering binds) and which both SDKs already classify as
4606    /// retryable with capped backoff. Reusing it is what keeps this change out
4607    /// of both SDKs; the daemon-side distinction lives in the counter key
4608    /// instead.
4609    ///
4610    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4611    /// While a breaker is open this fires on every open to that module, and the
4612    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4613    /// produced 261 lines about a single module inside 3000 lines of daemon
4614    /// log. The rare transitions are logged at warn/info instead and the volume
4615    /// is carried by the counter, so the evidence survives without the flood.
4616    /// The debug line keeps a per-refusal record reachable for whoever turns
4617    /// the level up.
4618    #[allow(clippy::too_many_arguments)]
4619    fn route_open_breaker_refusal_frame(
4620        &self,
4621        ctx: &RouteCtx,
4622        caller: &RouteOpenCaller,
4623        frame: &Frame,
4624        module_id: &str,
4625        consecutive_timeouts: u32,
4626        retry_in: Duration,
4627        probe_in_flight: bool,
4628    ) -> Result<Frame, RouterError> {
4629        self.counters
4630            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4631        debug!(
4632            target: "control",
4633            code = "module_timeout",
4634            module_id = ?module_id,
4635            connection_id = ctx.connection_id.get(),
4636            principal = %caller.principal,
4637            scope = %caller.scope,
4638            harness = %caller.harness,
4639            consecutive_timeouts,
4640            retry_in_ms = retry_in.as_millis() as u64,
4641            probe_in_flight,
4642            "route.open refused by open bind-relay breaker"
4643        );
4644        // Say what a caller can act on. An open bind-relay breaker means the
4645        // module timed out accepting several new routes in a row. The module
4646        // is still running and its established routes keep working; only new
4647        // route.open requests are refused until the cooldown ends and one
4648        // test route (the probe) gets through. A message that only counts
4649        // failed relays reads as "the module is down" to a worker that sees it.
4650        let detail = if probe_in_flight {
4651            "one test route is already being tried; retry once it settles".to_string()
4652        } else {
4653            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4654        };
4655        control_error_frame(
4656            frame,
4657            "module_timeout",
4658            format!(
4659                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4660                 timed out in a row); {detail}; its established routes are unaffected"
4661            ),
4662        )
4663    }
4664
4665    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4666    /// requester's bytes (an unknown target is whatever the client sent) and
4667    /// is Debug-formatted so control characters land in the log escaped
4668    /// rather than as terminal sequences for whoever tails it.
4669    ///
4670    /// `reason` names the check that refused, because one wire code has
4671    /// several senders: after a module registers, `target_unavailable` can
4672    /// come from a missing role, an inactive registration, a supervisor that
4673    /// has not marked the process live, a missing forwarding connection, or a
4674    /// failed relay, and a log that records only the code cannot say which of
4675    /// them fired. It is a static, daemon-chosen label per branch, so it is
4676    /// safe to print plainly and stays a closed set.
4677    fn observe_route_open_refusal(
4678        &self,
4679        ctx: &RouteCtx,
4680        caller: &RouteOpenCaller,
4681        module_id: &str,
4682        reason: &'static str,
4683        code: &'static str,
4684    ) {
4685        self.counters.increment_route_open_refused(code);
4686        info!(
4687            target: "control",
4688            code,
4689            reason,
4690            module_id = ?module_id,
4691            connection_id = ctx.connection_id.get(),
4692            principal = %caller.principal,
4693            scope = %caller.scope,
4694            harness = %caller.harness,
4695            "route.open refused"
4696        );
4697        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4698            self.route_outages.record_not_serving(module_id, reason);
4699        }
4700    }
4701
4702    /// Record an ACCEPTED route.open.
4703    ///
4704    /// Refusals have been logged and counted since the attestation work; accepts
4705    /// were invisible, so the daemon knew every principal it stamped and wrote
4706    /// none of them down. The party that attests the identity was the only party
4707    /// not recording it, which left a credential vault unable to name the sender
4708    /// of a call that reached it (claustrum #43) and left the launch-nonce
4709    /// concurrency question unanswerable from the outside.
4710    ///
4711    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4712    /// `code`/`module_id`/`connection_id` returns both directions of the same
4713    /// decision rather than two shapes a reader has to join by hand.
4714    ///
4715    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4716    /// and the difference carries information rather than being an
4717    /// inconsistency. This line is only reachable after a successful bind to a
4718    /// REGISTERED module, so the value has already passed HELLO validation
4719    /// including the path-hazard refusal and cannot contain control bytes. A
4720    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4721    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4722    ///
4723    /// Bare is also what every other daemon line already emits (`module
4724    /// registered`, `configured module supervised`). Shipping `?module_id` here
4725    /// made this instrument the only one in the file whose ids did not answer
4726    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4727    /// line whose whole purpose is being grepped beside its sibling.
4728    ///
4729    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4730    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4731    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4732    /// one are recorded identically. A test written against that harness passes
4733    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4734    /// green assertion that cannot fail. The same limit applies to the escaping
4735    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4736    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4737    /// Fencing either needs the real formatter, not the capture layer.
4738    ///
4739    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4740    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4741    /// to record. The ephemeral port would decay within minutes and answer only a
4742    /// live question. The identity question is instead answered by counting
4743    /// distinct live connections presenting one module's `consumer_identity` --
4744    /// "is anyone else holding this secret" rather than "is this the right
4745    /// process".
4746    fn observe_route_open_accept(
4747        &self,
4748        ctx: &RouteCtx,
4749        module_id: &str,
4750        principal: &str,
4751        scope: &str,
4752    ) {
4753        self.route_outages.record_accepted(module_id);
4754        self.counters.increment_route_open_accepted(principal);
4755        info!(
4756            target: "control",
4757            principal,
4758            module_id,
4759            connection_id = ctx.connection_id.get(),
4760            scope = %scope,
4761            "route.open accepted"
4762        );
4763    }
4764
4765    fn supervised_absent_route_open_refusal_frame(
4766        &self,
4767        ctx: &RouteCtx,
4768        caller: &RouteOpenCaller,
4769        frame: &Frame,
4770        module_id: &str,
4771        code: &'static str,
4772        status: &crate::supervise::ModuleStatus,
4773    ) -> Result<Frame, RouterError> {
4774        self.counters.increment_route_open_refused(code);
4775        info!(
4776            target: "control",
4777            code,
4778            reason = "supervised_not_registered",
4779            module_id = ?module_id,
4780            connection_id = ctx.connection_id.get(),
4781            principal = %caller.principal,
4782            scope = %caller.scope,
4783            harness = %caller.harness,
4784            state = %status.state,
4785            enabled = status.enabled,
4786            live = status.live,
4787            "route.open refused"
4788        );
4789        // A supervised module whose process has not registered is not
4790        // serving, whatever the reason; the supervisor knows this id, so it is
4791        // safe to track.
4792        self.route_outages
4793            .record_not_serving(module_id, "supervised_not_registered");
4794        let message = if let Some(pid) = status.waiting_for_previous_pid {
4795            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4796                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4797            })
4798        } else {
4799            format!(
4800                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4801                status.state, status.enabled, status.live
4802            )
4803        };
4804        control_error_frame(frame, code, message)
4805    }
4806
4807    async fn handle_route_open(
4808        &self,
4809        ctx: &RouteCtx,
4810        frame: Frame,
4811        request: RouteOpenRequest,
4812        timings: &mut RouteOpenTimings,
4813    ) -> Result<Vec<Frame>, RouterError> {
4814        let RouteOpenRequest {
4815            target,
4816            mut identity,
4817            consumer_identity,
4818            consumer_capabilities,
4819            role_versions,
4820            busy_refusals,
4821            admission_facts,
4822            scope,
4823        } = request;
4824        let target_module_id = target_module_id(&target).to_string();
4825        timings.module_id.clone_from(&target_module_id);
4826        if self
4827            .registry
4828            .get_module_by_connection(ctx.connection_id)
4829            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4830            .is_some()
4831        {
4832            return Ok(vec![control_error_frame(
4833                &frame,
4834                "invalid_request",
4835                "module connections cannot open client routes",
4836            )?]);
4837        }
4838        debug!(
4839            connection_id = ctx.connection_id.get(),
4840            corr = frame.header.corr,
4841            module_id = %target_module_id,
4842            "handling route.open"
4843        );
4844
4845        // The caller's attested principal, requested scope and harness, for
4846        // every `route.open refused` line below. The launch nonce is checked
4847        // here, once, and the answer is reused where the principal is
4848        // enforced further down, so the order of refusals is unchanged.
4849        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4850        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4851
4852        // A malformed declaration is refused first, before anything about the
4853        // target is looked up: the same body would be refused against any
4854        // module, so the caller learns nothing by retrying or waiting. An empty
4855        // map declares nothing and travels as no field at all, so a provider
4856        // only ever sees a missing field or a non-empty one.
4857        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4858        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4859            self.observe_route_open_refusal(
4860                ctx,
4861                &caller,
4862                &target_module_id,
4863                "invalid_role_versions",
4864                error_codes::INVALID_REQUEST,
4865            );
4866            return Ok(vec![control_error_body_frame(
4867                &frame,
4868                ErrorBody {
4869                    code: error_codes::INVALID_REQUEST.to_string(),
4870                    message: error.to_string(),
4871                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4872                },
4873            )?]);
4874        }
4875
4876        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4877            if status.waiting_for_previous_pid.is_some() {
4878                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4879                    ctx,
4880                    &caller,
4881                    &frame,
4882                    &target_module_id,
4883                    "module_warming",
4884                    &status,
4885                )?]);
4886            }
4887        }
4888
4889        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4890        // opposite. Below, a caller learns whether a module is unregistered,
4891        // supervised-but-down (with state/enabled/live), or registered without the
4892        // requested role. Elsewhere that is an enumeration leak: a probe learning
4893        // the shape of a fleet it cannot otherwise see.
4894        //
4895        // It is not one here, and the reason is the ACCESS MODEL rather than
4896        // anything about these errors. Reaching route.open requires the
4897        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4898        // connection file, so any caller who completes it already runs as this
4899        // user -- and can read subc.jsonc for the module list and `ck module
4900        // status` for live state. The reply discloses nothing the caller cannot
4901        // read more easily from disk, while the precision is load-bearing:
4902        // `unknown_module` is retryable and a missing role is not.
4903        //
4904        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4905        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4906        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4907        let Some(registration) = self
4908            .registry
4909            .get_module(&target_module_id)
4910            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4911        else {
4912            if let Some((status, warming)) =
4913                self.supervisor_status(&target_module_id, frame.header.corr)?
4914            {
4915                // BEFORE the two availability codes below, because for a module
4916                // that speaks no subc wire both of them are false comfort: they
4917                // say "not right now" and are retried, and this module will
4918                // never register no matter how long the caller waits. The
4919                // absence here is the declaration being honoured, not a module
4920                // that is late.
4921                if status.protocol == ModuleProtocol::None {
4922                    return Ok(vec![self.route_open_refusal_frame(
4923                        ctx,
4924                        &caller,
4925                        &frame,
4926                        &target_module_id,
4927                        "protocol_none",
4928                        error_codes::MODULE_NO_PROTOCOL,
4929                        format!(
4930                            "module_id '{target_module_id}' is declared protocol: none; \
4931                             it speaks no subc wire and serves no routes"
4932                        ),
4933                    )?]);
4934                }
4935                let code = if warming {
4936                    "module_warming"
4937                } else {
4938                    "target_unavailable"
4939                };
4940                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4941                    ctx,
4942                    &caller,
4943                    &frame,
4944                    &target_module_id,
4945                    code,
4946                    &status,
4947                )?]);
4948            }
4949            if let Some(removed_ago_ms) =
4950                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4951            {
4952                return Ok(vec![self.route_open_refusal_frame(
4953                    ctx,
4954                    &caller,
4955                    &frame,
4956                    &target_module_id,
4957                    "removed",
4958                    error_codes::MODULE_REMOVED,
4959                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4960                )?]);
4961            }
4962            return Ok(vec![self.route_open_refusal_frame(
4963                ctx,
4964                &caller,
4965                &frame,
4966                &target_module_id,
4967                "not_registered",
4968                error_codes::UNKNOWN_MODULE,
4969                format!("module_id '{target_module_id}' is not registered"),
4970            )?]);
4971        };
4972
4973        // Best-effort only: registry readiness and forwarding reservation use
4974        // different locks, so a module can flip readiness between this read and
4975        // the relay. Modules must still tolerate an `on_bind` while not ready.
4976        if !registration.ready {
4977            self.counters
4978                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4979            info!(
4980                target: "control",
4981                code = error_codes::MODULE_WARMING,
4982                module_id = ?target_module_id,
4983                connection_id = ctx.connection_id.get(),
4984                principal = %caller.principal,
4985                scope = %caller.scope,
4986                harness = %caller.harness,
4987                reason = "declared_not_ready",
4988                "route.open refused"
4989            );
4990            // The module is registered but says it cannot take work, which is
4991            // an outage from the caller's side even though its process is up.
4992            self.route_outages
4993                .record_not_serving(&target_module_id, "declared_not_ready");
4994            return Ok(vec![control_error_body_frame(
4995                &frame,
4996                ErrorBody {
4997                    code: error_codes::MODULE_WARMING.to_string(),
4998                    message: format!(
4999                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5000                    ),
5001                    detail: Some(serde_json::json!({
5002                        "reason": "declared_not_ready"
5003                    })),
5004                },
5005            )?]);
5006        }
5007
5008        // Effective readiness, second half: a module that declares a capability
5009        // `need: required` is not routable while that capability has no
5010        // registered provider. It is enforced HERE, as a retryable routing
5011        // refusal, and deliberately not as spawn ordering or a boot block. The
5012        // module is still started and registered and can make its own calls;
5013        // spawn ordering is a promise that cannot be kept once a provider
5014        // crashes at runtime, and refusing to boot would stop the whole
5015        // machine, including the tools needed to fix its configuration.
5016        //
5017        // "Provided" is the evaluator's verdict, which counts a provider as
5018        // soon as it has REGISTERED, not once it is ready. Two modules that
5019        // require each other's capabilities are therefore both routable once
5020        // both register; counting readiness instead would deadlock them.
5021        //
5022        // Only new opens are refused. Routes already bound when a provider
5023        // goes away stay bound: nothing here tears them down, and the module
5024        // answers them as it can. Like the readiness read above this is
5025        // best-effort against a provider registering or leaving concurrently.
5026        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
5027            self.counters
5028                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5029            info!(
5030                target: "control",
5031                code = error_codes::MODULE_WARMING,
5032                module_id = ?target_module_id,
5033                connection_id = ctx.connection_id.get(),
5034                principal = %caller.principal,
5035                scope = %caller.scope,
5036                harness = %caller.harness,
5037                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5038                capability = %capability,
5039                "route.open refused"
5040            );
5041            return Ok(vec![control_error_body_frame(
5042                &frame,
5043                ErrorBody {
5044                    code: error_codes::MODULE_WARMING.to_string(),
5045                    message: format!(
5046                        "module_id '{target_module_id}' requires capability '{capability}', \
5047                         which no registered module provides; retry"
5048                    ),
5049                    detail: Some(serde_json::json!({
5050                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5051                        "capability": capability,
5052                    })),
5053                },
5054            )?]);
5055        }
5056
5057        if !target_has_required_role(&target, &registration.manifest.provides) {
5058            return Ok(vec![self.route_open_refusal_frame(
5059                ctx,
5060                &caller,
5061                &frame,
5062                &target_module_id,
5063                "role_not_provided",
5064                "target_unavailable",
5065                format!("module_id '{target_module_id}' does not provide the requested target"),
5066            )?]);
5067        }
5068
5069        if registration.state != ChannelState::Active {
5070            return Ok(vec![self.route_open_refusal_frame(
5071                ctx,
5072                &caller,
5073                &frame,
5074                &target_module_id,
5075                "registration_not_active",
5076                "target_unavailable",
5077                format!("module_id '{target_module_id}' is not active"),
5078            )?]);
5079        }
5080
5081        if self
5082            .forwarding
5083            .module_is_draining(&target_module_id)
5084            .map_err(RouterError::Forwarding)?
5085        {
5086            return Ok(vec![self.route_open_refusal_frame(
5087                ctx,
5088                &caller,
5089                &frame,
5090                &target_module_id,
5091                "reloading",
5092                "module_reloading",
5093                format!("module_id '{target_module_id}' is reloading"),
5094            )?]);
5095        }
5096
5097        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5098            process_liveness.process_live(&target_module_id) == Some(false)
5099        }) {
5100            // A module the supervisor is restarting or reloading can still hold
5101            // a registration: the old process before its connection closes, or
5102            // a new one that registered while the supervisor was draining. The
5103            // forwarding table does not see that as draining, but the consumer
5104            // should still be told to retry soon, exactly as for the drain
5105            // above, rather than that the target is unavailable.
5106            if process_liveness.process_replacing(&target_module_id) {
5107                return Ok(vec![self.route_open_refusal_frame(
5108                    ctx,
5109                    &caller,
5110                    &frame,
5111                    &target_module_id,
5112                    "reloading",
5113                    "module_reloading",
5114                    format!("module_id '{target_module_id}' is reloading"),
5115                )?]);
5116            }
5117            return Ok(vec![self.route_open_refusal_frame(
5118                ctx,
5119                &caller,
5120                &frame,
5121                &target_module_id,
5122                "supervisor_not_live",
5123                "target_unavailable",
5124                format!("module_id '{target_module_id}' is not live"),
5125            )?]);
5126        }
5127
5128        if !self
5129            .forwarding
5130            .has_live_module_connection(&target_module_id)
5131            .map_err(RouterError::Forwarding)?
5132        {
5133            return Ok(vec![self.route_open_refusal_frame(
5134                ctx,
5135                &caller,
5136                &frame,
5137                &target_module_id,
5138                "no_forwarding_connection",
5139                "target_unavailable",
5140                format!("module_id '{target_module_id}' has no live forwarding connection"),
5141            )?]);
5142        }
5143
5144        if let Some(error) =
5145            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5146        {
5147            self.observe_route_open_refusal(
5148                ctx,
5149                &caller,
5150                &target_module_id,
5151                "op_not_allowed",
5152                "op_not_allowed",
5153            );
5154            return Ok(vec![error]);
5155        }
5156
5157        let principal =
5158            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5159                Ok(principal) => principal,
5160                Err(error) => {
5161                    self.observe_route_open_refusal(
5162                        ctx,
5163                        &caller,
5164                        &target_module_id,
5165                        "bad_consumer_identity",
5166                        "bad_consumer_identity",
5167                    );
5168                    return Ok(vec![error]);
5169                }
5170            };
5171        timings.principal = principal_label(&principal);
5172
5173        // Only Callosum can attest a production phone build. Consume the claim
5174        // here so it can be kept on the route without reaching any module.
5175        let build_class = identity.build_class.take().filter(|build_class| {
5176            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5177                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5178            } else {
5179                if self
5180                    .build_class_warned
5181                    .lock()
5182                    .unwrap_or_else(|p| p.into_inner())
5183                    .insert(ctx.connection_id)
5184                {
5185                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5186                        "ignoring build_class from a non-Callosum opener");
5187                }
5188                false
5189            }
5190        });
5191
5192        // This is attested, control-plane policy for supervised module origins.
5193        // Keep it before route reservation and out of the opaque forwarding hot
5194        // path: data frames must never acquire a per-frame capability check.
5195        if let Principal::Reserved {
5196            module_id: opening_module_id,
5197        } = &principal
5198        {
5199            if let Some(opening_registration) = self
5200                .registry
5201                .get_module(opening_module_id)
5202                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5203            {
5204                if let Some(capability) =
5205                    denied_capability(&opening_registration.manifest, &registration.manifest)
5206                {
5207                    warn!(
5208                        opening_module_id,
5209                        target_module_id,
5210                        capability,
5211                        "refusing route.open because an attested capability deny edge matches"
5212                    );
5213                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5214                        &frame,
5215                        &target_module_id,
5216                        "capability_deny_edge",
5217                        "capability_forbidden",
5218                        format!(
5219                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5220                        ),
5221                    )?]);
5222                }
5223            }
5224        }
5225
5226        if admission_facts.is_some() {
5227            let carrier_matches = matches!(
5228                &principal,
5229                Principal::Reserved { module_id }
5230                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5231            );
5232            if !carrier_matches {
5233                return Ok(vec![self.route_open_refusal_frame(
5234                    ctx,
5235                    &caller,
5236                    &frame,
5237                    &target_module_id,
5238                    "admission_facts_carrier_not_permitted",
5239                    "admission_facts_not_permitted",
5240                    "admission facts may only be carried by the configured reserved module",
5241                )?]);
5242            }
5243
5244            let target_allowed = self
5245                .admission_facts_targets
5246                .as_ref()
5247                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5248            if !target_allowed {
5249                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5250                    &frame,
5251                    &target_module_id,
5252                    "admission_facts_target_not_listed",
5253                    "admission_facts_target_not_allowed",
5254                    format!(
5255                        "admission facts are not permitted for target module_id '{target_module_id}'"
5256                    ),
5257                )?]);
5258            }
5259
5260            // Keep the value opaque to subc. The downstream admission validator owns
5261            // schema and semantic checks; this daemon only enforces carrier authority
5262            // and the configured destination allowlist.
5263        }
5264
5265        // Scope admission, on the attested principal above and never on the
5266        // request body. The tag read here travels with the pending bind and is
5267        // compared with the published one at commit, so a sync between here
5268        // and the module's ack refuses the open instead of binding a stamp
5269        // that is no longer true.
5270        let (bound_scope, mut scope_stamp) = match &scope {
5271            None => (None, None),
5272            Some(selector) => {
5273                let owner_configured = match &selector.owner {
5274                    // A reserved owner counts as configured from before its
5275                    // process is spawned (see `SupervisorHandle::is_configured`).
5276                    // So an owner that has not synced its scopes yet is refused
5277                    // as retryable (`scope_not_synced`), not as one that will
5278                    // never sync.
5279                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5280                    _ => false,
5281                };
5282                let mut scopes = self
5283                    .scopes
5284                    .write()
5285                    .unwrap_or_else(|poisoned| poisoned.into_inner());
5286                let admitted = scopes
5287                    .admit(&principal, &target_module_id, selector, owner_configured)
5288                    .and_then(|admission| {
5289                        crate::scopes::check_target_flow_support(
5290                            &admission.stamp,
5291                            &target_module_id,
5292                            registration.manifest.capabilities.as_ref(),
5293                        )?;
5294                        crate::scopes::check_target_agent_run_support(
5295                            &admission.stamp,
5296                            &target_module_id,
5297                            registration.manifest.capabilities.as_ref(),
5298                        )?;
5299                        let bound = scopes.reserve_bind(
5300                            &admission,
5301                            &principal,
5302                            ctx.connection_id,
5303                            selector.hold_presence,
5304                        );
5305                        Ok((admission, bound))
5306                    });
5307                match admitted {
5308                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5309                    Err(refusal) => {
5310                        return Ok(vec![self.route_open_refusal_frame(
5311                            ctx,
5312                            &caller,
5313                            &frame,
5314                            &target_module_id,
5315                            refusal.code,
5316                            refusal.code,
5317                            refusal.message,
5318                        )?]);
5319                    }
5320                }
5321            }
5322        };
5323        // From here on a refusal names the scope by the stamp admission
5324        // issued, exactly as the `route.open accepted` line does.
5325        if let Some(stamp) = &scope_stamp {
5326            caller.scope = route_open_scope_label(stamp);
5327        }
5328
5329        // Allow resolvable missing-tail paths so a vanished directory does not
5330        // prevent reopening an existing session for cleanup. Binding does not
5331        // authorize creation of durable provider state.
5332        //
5333        // Resolve real paths rather than lexical cleanup: canonicalize the
5334        // longest existing ancestor and append the missing tail, retaining the
5335        // path's real existing ancestry even when a leaf is missing.
5336        timings.enter(RouteOpenPhase::ProjectRoot);
5337        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5338            Ok(project) => project,
5339            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5340        };
5341        timings.enter(RouteOpenPhase::Admission);
5342        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5343            Ok(root) => root,
5344            Err(err) => {
5345                return Ok(vec![control_error_frame(
5346                    &frame,
5347                    "invalid_project_root",
5348                    err.to_string(),
5349                )?])
5350            }
5351        };
5352        // Compute compatibility project fields before stripping optional stamp
5353        // members. Preserve the canonical opener root and its project ID,
5354        // including absence. For rootless openers, use stamp workdir or `/`, and
5355        // opener ID, else stamp ID, else no ID. Emit these identity fields only
5356        // to targets without bind-no-project/v1.
5357        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5358
5359        // Last gate before any relay work, and deliberately after the cheap
5360        // registry and availability checks above: those name a more precise
5361        // condition (unknown, removed, reloading) and a caller is better served
5362        // by the precise code than by this one.
5363        //
5364        // Everything below this point costs an egress permit, a reserved handle
5365        // pair and, if the module does not answer, the whole relay budget. The
5366        // reader no longer waits for that budget, so cap each target explicitly;
5367        // serial dispatch used to provide the accidental cap of one relay per
5368        // connection. Admission is a mutex-protected count and never waits.
5369        let _concurrency_guard = match self
5370            .route_bind_concurrency
5371            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5372        {
5373            Ok(guard) => guard,
5374            Err(in_flight) => {
5375                return Ok(vec![self.route_open_target_capacity_refusal(
5376                    ctx,
5377                    &caller,
5378                    &frame,
5379                    &target_module_id,
5380                    in_flight,
5381                )?]);
5382            }
5383        };
5384
5385        // A module that has already burned the whole budget `threshold` times
5386        // in a row does not get to charge it again until a probe says it recovered.
5387        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5388            RouteBindAdmission::Admitted { guard, probe } => {
5389                if probe {
5390                    info!(
5391                        module_id = %target_module_id,
5392                        connection_id = ctx.connection_id.get(),
5393                        "route.bind breaker half-open: admitting one probe"
5394                    );
5395                }
5396                guard
5397            }
5398            RouteBindAdmission::Refused {
5399                consecutive_timeouts,
5400                retry_in,
5401                probe_in_flight,
5402            } => {
5403                return Ok(vec![self.route_open_breaker_refusal_frame(
5404                    ctx,
5405                    &caller,
5406                    &frame,
5407                    &target_module_id,
5408                    consecutive_timeouts,
5409                    retry_in,
5410                    probe_in_flight,
5411                )?]);
5412            }
5413        };
5414
5415        // Resolve the per-module budget here so the wait matches the operator's
5416        // intent for this specific target. A per-module override in
5417        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5418        // daemons) wins over the daemon-wide default.
5419        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5420        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5421        let operator_app = self
5422            .operator_apps
5423            .lock()
5424            .unwrap_or_else(|p| p.into_inner())
5425            .get(&ctx.connection_id)
5426            .cloned();
5427        timings.enter(RouteOpenPhase::EgressReservation);
5428        let pending = match self
5429            .forwarding
5430            .begin_route_bind_relay_for(
5431                ctx.connection_id,
5432                ctx.egress.clone(),
5433                response_version(&frame),
5434                frame.header.corr,
5435                &target_module_id,
5436                principal.clone(),
5437                bound_scope,
5438                project_root,
5439                identity.harness.clone(),
5440                build_class,
5441                operator_app.clone(),
5442                relay_deadline,
5443                busy_refusals == Some(true),
5444            )
5445            .await
5446        {
5447            Ok(pending) => pending,
5448            Err(err) => {
5449                return Ok(vec![self.route_open_refusal_frame(
5450                    ctx,
5451                    &caller,
5452                    &frame,
5453                    &target_module_id,
5454                    "relay_reservation_failed",
5455                    forwarding_error_code(&err),
5456                    err.to_string(),
5457                )?])
5458            }
5459        };
5460        timings.enter(RouteOpenPhase::Admission);
5461        let crate::forwarding::PendingRouteBindRelay {
5462            endpoint,
5463            module_sink,
5464            negotiated_ver,
5465            client_channel,
5466            client_epoch,
5467            module_channel,
5468            module_epoch,
5469            corr: relay_corr,
5470            receiver,
5471        } = pending;
5472        let mut reservation =
5473            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5474
5475        // Reserving egress can wait while a module reconnects or a swap cuts
5476        // over. Check the relay's captured connection, not the earlier by-id
5477        // lookup: the original module's flow/run capabilities cannot authorize
5478        // its replacement. The captured sink stays bound to that connection.
5479        let relay_registration = self
5480            .registry
5481            .get_module_by_connection(endpoint.connection_id)
5482            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5483        let relay_capabilities = relay_registration
5484            .as_ref()
5485            .and_then(|registration| registration.manifest.capabilities.as_ref());
5486        if let Some(selector) = &scope {
5487            // Egress reservation may wait across a scope update. Recheck the
5488            // proof against the current record before sending the captured bind.
5489            if let Err(refusal) = self
5490                .scopes
5491                .read()
5492                .unwrap_or_else(|poisoned| poisoned.into_inner())
5493                .check_bind_proof(&principal, selector)
5494            {
5495                reservation.release_and_disarm();
5496                return Ok(vec![self.route_open_refusal_frame(
5497                    ctx,
5498                    &caller,
5499                    &frame,
5500                    &target_module_id,
5501                    refusal.code,
5502                    refusal.code,
5503                    refusal.message,
5504                )?]);
5505            }
5506        }
5507        if let Some(stamp) = scope_stamp
5508            .as_ref()
5509            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5510        {
5511            if let Err(refusal) = crate::scopes::check_target_flow_support(
5512                stamp,
5513                &target_module_id,
5514                relay_capabilities,
5515            )
5516            .and_then(|()| {
5517                crate::scopes::check_target_agent_run_support(
5518                    stamp,
5519                    &target_module_id,
5520                    relay_capabilities,
5521                )
5522            }) {
5523                reservation.release_and_disarm();
5524                return Ok(vec![self.route_open_refusal_frame(
5525                    ctx,
5526                    &caller,
5527                    &frame,
5528                    &target_module_id,
5529                    refusal.code,
5530                    refusal.code,
5531                    refusal.message,
5532                )?]);
5533            }
5534        }
5535
5536        // Older readers may reject unknown fields. Select optional fields from
5537        // the captured endpoint's declaration, including on unscoped binds.
5538        if !relay_capabilities.is_some_and(|caps| {
5539            caps.provides
5540                .iter()
5541                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5542        }) {
5543            if let Some(stamp) = scope_stamp.as_mut() {
5544                stamp.workdir = None;
5545                stamp.project_id = None;
5546            }
5547        }
5548        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5549        // attribution. The run gate above has already required the target to
5550        // declare `worker-runs/v1`, which promises to read both
5551        // `on_behalf_of_agent` and `call_mode`.
5552        let worker_run = scope_stamp
5553            .as_ref()
5554            .is_some_and(crate::scopes::is_worker_run)
5555            && relay_capabilities.is_some_and(|caps| {
5556                caps.provides
5557                    .iter()
5558                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5559            });
5560        if !worker_run
5561            && !relay_capabilities.is_some_and(|caps| {
5562                caps.provides
5563                    .iter()
5564                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5565            })
5566        {
5567            if let Some(stamp) = scope_stamp.as_mut() {
5568                stamp.attributes.on_behalf_of_agent = None;
5569            }
5570        }
5571        if !relay_capabilities.is_some_and(|caps| {
5572            caps.provides
5573                .iter()
5574                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5575        }) {
5576            if let Some(stamp) = scope_stamp.as_mut() {
5577                stamp.attributes.flow_version = None;
5578                stamp.attributes.flow_code_hash = None;
5579                stamp.attributes.flow_install_id = None;
5580            }
5581        }
5582        let call_mode = if worker_run
5583            || relay_capabilities
5584                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5585        {
5586            Some(derive_call_mode(scope_stamp.as_ref()))
5587        } else {
5588            if let Some(stamp) = scope_stamp.as_mut() {
5589                stamp.expires_at_ms = None;
5590            }
5591            None
5592        };
5593
5594        debug!(
5595            connection_id = ctx.connection_id.get(),
5596            client_channel,
5597            client_epoch,
5598            module_channel,
5599            module_epoch,
5600            "reserved route handle pair"
5601        );
5602        // Rendered BEFORE the move into the relay, because the accept arm below
5603        // is where it is logged and the principal is gone by then.
5604        let route_principal_label = principal_label(&principal);
5605        let relay = ModuleControlRequest::RouteBind {
5606            route_channel: module_channel,
5607            epoch: module_epoch,
5608            target,
5609            identity,
5610            principal: Some(principal),
5611            consumer_capabilities,
5612            role_versions,
5613            admission_facts,
5614            scope: scope_stamp,
5615            call_mode,
5616        };
5617        let peer = RouteBindPeer::new()
5618            .with_observed_peer(
5619                relay_capabilities
5620                    .is_some_and(|caps| {
5621                        caps.provides
5622                            .iter()
5623                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5624                    })
5625                    .then(|| {
5626                        self.observed_peers
5627                            .lock()
5628                            .unwrap_or_else(|p| p.into_inner())
5629                            .get(&ctx.connection_id)
5630                            .cloned()
5631                            .unwrap_or_else(ObservedPeer::unobserved)
5632                    }),
5633            )
5634            .with_operator_app(
5635                self.connection_operator_app(operator_app.as_ref())
5636                    .filter(|_| {
5637                        relay_capabilities.is_some_and(|caps| {
5638                            caps.provides
5639                                .iter()
5640                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5641                        })
5642                    }),
5643            );
5644        let relay_body = crate::bind_compat::encode_bind(
5645            &relay,
5646            &peer,
5647            &legacy_bind_project,
5648            relay_capabilities,
5649        )
5650        .map_err(|err| {
5651            RouterError::backend(
5652                0,
5653                frame.header.corr,
5654                format!("failed to encode route.bind request: {err}"),
5655            )
5656        })?;
5657        let relay_frame = Frame::build_with_version(
5658            negotiated_ver,
5659            FrameType::Request,
5660            control_flags(),
5661            0,
5662            0,
5663            relay_corr,
5664            relay_body,
5665        )
5666        .map_err(RouterError::FrameBuild)?;
5667
5668        timings.enter(RouteOpenPhase::BindSend);
5669        let sent = module_sink.send(relay_frame).await;
5670        timings.enter(RouteOpenPhase::Admission);
5671        if let Err(err) = sent {
5672            reservation.release_and_disarm();
5673            return Ok(vec![self.route_open_refusal_frame(
5674                ctx,
5675                &caller,
5676                &frame,
5677                &target_module_id,
5678                "relay_send_failed",
5679                "target_unavailable",
5680                err.to_string(),
5681            )?]);
5682        }
5683
5684        #[cfg(test)]
5685        if let Some(hold) = &self.route_bind_relay_sent_hold {
5686            // First wait: tell the test the bind is queued. Second wait: stay
5687            // here until the test has driven whatever it wants to happen
5688            // between queueing the bind and marking it enqueued.
5689            hold.wait().await;
5690            hold.wait().await;
5691        }
5692
5693        // A missing relay is not by itself an abandoned bind: the module may
5694        // already have acked it and the route may be committed. Only a relay
5695        // dropped without any module reply needs this GOODBYE; one a reply
5696        // settled has already had its GOODBYE decided by the reply path.
5697        if self
5698            .forwarding
5699            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5700            .map_err(RouterError::Forwarding)?
5701            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5702        {
5703            self.send_abandoned_route_bind_goodbye(
5704                &module_sink,
5705                negotiated_ver,
5706                module_channel,
5707                module_epoch,
5708            );
5709        }
5710
5711        timings.enter(RouteOpenPhase::BindAckWait);
5712        let outcome = timeout_at(relay_deadline, receiver).await;
5713        timings.enter(RouteOpenPhase::Admission);
5714        match outcome {
5715            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5716                reservation.disarm();
5717                if breaker.record_accepted() {
5718                    info!(
5719                        module_id = %target_module_id,
5720                        "route.bind breaker closed: the probe was accepted"
5721                    );
5722                }
5723                self.observe_route_open_accept(
5724                    ctx,
5725                    &target_module_id,
5726                    &route_principal_label,
5727                    &caller.scope,
5728                );
5729                Ok(Vec::new())
5730            }
5731            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5732                reservation.release_and_disarm();
5733                // A module that says no in microseconds is healthy. Rejection
5734                // is a different condition with its own refusal and must not
5735                // move the breaker.
5736                breaker.record_inconclusive();
5737                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5738            }
5739            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5740                reservation.release_and_disarm();
5741                breaker.record_inconclusive();
5742                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5743            }
5744            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5745                reservation.release_and_disarm();
5746                breaker.record_inconclusive();
5747                // Fires when the module's connection closes while a relayed
5748                // bind is pending -- typically a caller racing a module restart
5749                // whose bind was relayed BEFORE the drain mark went up. Logged
5750                // because the caller sees only its own error and the fleet has
5751                // already spent one diagnosis round unable to tell this arm
5752                // from a relay timeout without daemon-side evidence.
5753                tracing::warn!(
5754                    module_id = %target_module_id,
5755                    "route.bind relay abandoned: {message}"
5756                );
5757                Ok(vec![self.route_open_refusal_frame(
5758                    ctx,
5759                    &caller,
5760                    &frame,
5761                    &target_module_id,
5762                    "relay_abandoned",
5763                    "target_unavailable",
5764                    message,
5765                )?])
5766            }
5767            Ok(Err(_)) => {
5768                reservation.release_and_disarm();
5769                breaker.record_inconclusive();
5770                Ok(vec![self.route_open_refusal_frame(
5771                    ctx,
5772                    &caller,
5773                    &frame,
5774                    &target_module_id,
5775                    "relay_waiter_canceled",
5776                    "target_unavailable",
5777                    "route.bind relay waiter was canceled before the module responded",
5778                )?])
5779            }
5780            Err(_) => {
5781                reservation.release_and_disarm();
5782                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5783                // answer at all is the one condition a fast refusal can
5784                // usefully stand in for; every other arm already answered.
5785                if let Some(opened) = breaker.record_timeout(
5786                    self.route_bind_breaker_threshold,
5787                    self.route_bind_breaker_cooldown,
5788                ) {
5789                    warn!(
5790                        module_id = %target_module_id,
5791                        consecutive_timeouts = opened.consecutive_timeouts,
5792                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5793                        reopened_after_probe = opened.reopened_after_probe,
5794                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5795                    );
5796                }
5797                // The generous budget just burned to no answer: the module is
5798                // registered and its connection is up, but its bind handler sat
5799                // on the ack for the full budget (warm-on-bind, cold configure,
5800                // or a wedged handler). Every earlier unavailability shape
5801                // fast-refuses BEFORE the relay, so this arm firing means the
5802                // slowness is module-side -- log it so the per-module timeline
5803                // is reconstructable without client audit rows.
5804                tracing::warn!(
5805                    module_id = %target_module_id,
5806                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5807                    "route.bind relay timed out: module did not ack within budget"
5808                );
5809                Ok(vec![self.route_open_refusal_frame(
5810                    ctx,
5811                    &caller,
5812                    &frame,
5813                    &target_module_id,
5814                    "relay_timed_out",
5815                    "module_timeout",
5816                    format!(
5817                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5818                        route_bind_relay_timeout
5819                    ),
5820                )?])
5821            }
5822        }
5823    }
5824
5825    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5826        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5827            snapshot: self.supervisor.spawn_snapshot(),
5828        };
5829        Ok(vec![control_response_body_frame(
5830            &frame,
5831            &response,
5832            "ClientControlResponse::SupervisorSpawnSnapshot",
5833        )?])
5834    }
5835
5836    fn handle_supervisor_spawn_subscribe(
5837        &self,
5838        ctx: &RouteCtx,
5839        frame: Frame,
5840        since: Option<SpawnCursor>,
5841    ) -> Result<Vec<Frame>, RouterError> {
5842        match self.supervisor.subscribe_spawns(
5843            ctx.connection_id,
5844            frame.header.corr,
5845            response_version(&frame),
5846            since,
5847            ctx.egress.clone(),
5848        ) {
5849            Ok(()) => Ok(Vec::new()),
5850            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5851                Ok(vec![control_error_body_frame(
5852                    &frame,
5853                    ErrorBody {
5854                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5855                        message: "spawn cursor belongs to a different daemon incarnation"
5856                            .to_string(),
5857                        detail: Some(serde_json::json!({
5858                            "current_daemon_incarnation": current
5859                        })),
5860                    },
5861                )?])
5862            }
5863            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5864                &frame,
5865                ErrorBody {
5866                    code: "spawn_cursor_too_old".to_string(),
5867                    message: "spawn cursor predates the retained event ring".to_string(),
5868                    detail: Some(serde_json::json!({
5869                        "oldest_retained_cursor": oldest
5870                    })),
5871                },
5872            )?]),
5873            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5874                0,
5875                frame.header.corr,
5876                format!("failed to open supervisor spawn subscription: {error}"),
5877            )),
5878        }
5879    }
5880
5881    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882        let generation = self
5883            .registry
5884            .generation()
5885            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5886        let supervised = self.supervisor.list();
5887        let targets = supervised
5888            .iter()
5889            .map(|module| module.subtree_target())
5890            .collect();
5891        let subtree = tokio::spawn(crate::subtree_resources::read(targets));
5892        let mut modules = Vec::new();
5893        for module in supervised {
5894            let status = module.status_for_control("list").map_err(|err| {
5895                RouterError::backend(
5896                    0,
5897                    frame.header.corr,
5898                    format!("failed to read supervisor status: {err}"),
5899                )
5900            })?;
5901            let (configured, _) = module.configuration().map_err(|err| {
5902                RouterError::backend(
5903                    0,
5904                    frame.header.corr,
5905                    format!("failed to read module configuration: {err}"),
5906                )
5907            })?;
5908            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5909                RouterError::backend(
5910                    0,
5911                    frame.header.corr,
5912                    format!("failed to read retry time: {err}"),
5913                )
5914            })?;
5915            // Status and configuration snapshots release their locks before the image probe awaits.
5916            let image = module.running_image_agreement().await;
5917            // Read per request so the figure is current when the operator asks;
5918            // the daemon samples nothing in between.
5919            let resources = Some(crate::subtree_resources::read_process(module.clone()).await);
5920            let mut verdict =
5921                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5922            verdict.spec = Some(launch_spec_agreement(
5923                &configured,
5924                status.process_alive,
5925                status.launch_spec_digest.as_ref(),
5926            ));
5927            let pending_reload = Some(verdict);
5928            let configured_enabled = self
5929                .capability_evaluator
5930                .configured_enabled(&status.module_id);
5931            // Keep the retired policy field on the wire for one release so
5932            // existing status consumers still receive the platform policy.
5933            modules.push(
5934                SupervisorEntry::new(
5935                    status.module_id,
5936                    status.state.to_string(),
5937                    status.enabled,
5938                    status.live,
5939                    status.health.status,
5940                )
5941                .with_configured_enabled(configured_enabled)
5942                .with_next_retry_at_ms(next_retry_at_ms)
5943                .with_launch_nonce_env(Some(!cfg!(unix)))
5944                .with_launch_nonce_source(status.launch_nonce_source)
5945                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5946                .with_protocol(status.protocol)
5947                .with_pending_reload(pending_reload)
5948                .with_last_probe_ms(status.health.last_probe_ms)
5949                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5950                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5951                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5952                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5953                .with_restart_count(Some(status.restart_count))
5954                .with_max_restarts(Some(status.max_restarts))
5955                .with_lifetime_restarts(Some(status.lifetime_restarts))
5956                .with_spawn_generation(Some(status.spawn_generation))
5957                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5958                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5959                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5960                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5961                .with_resources(resources),
5962            );
5963        }
5964        let readings = subtree
5965            .await
5966            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5967        for (module, reading) in modules.iter_mut().zip(readings) {
5968            if reading.reason.as_deref() == Some("timeout") {
5969                crate::subtree_resources::log_timeout(&module.module_id, "subtree");
5970            }
5971            module.subtree = Some(reading);
5972        }
5973        let response = ClientControlResponse::SupervisorList {
5974            generation,
5975            modules,
5976        };
5977        Ok(vec![control_response_body_frame(
5978            &frame,
5979            &response,
5980            "ClientControlResponse::SupervisorList",
5981        )?])
5982    }
5983
5984    fn handle_supervisor_stderr_tail(
5985        &self,
5986        frame: Frame,
5987        module_id: String,
5988        max_lines: Option<u32>,
5989        max_bytes: Option<u32>,
5990    ) -> Result<Vec<Frame>, RouterError> {
5991        let Some(module) = self.supervisor.get(&module_id) else {
5992            return Ok(vec![control_error_frame(
5993                &frame,
5994                "unknown_module",
5995                format!("module_id '{module_id}' is not supervised"),
5996            )?]);
5997        };
5998
5999        let snapshot = module.stderr_tail(
6000            max_lines.map(|value| value as usize),
6001            max_bytes.map(|value| value as usize),
6002        );
6003
6004        let response = ClientControlResponse::SupervisorStderrTail {
6005            module_id,
6006            tail: StderrTail {
6007                capture: match snapshot.capture {
6008                    CaptureState::Captured => StderrCaptureState::Captured,
6009                    CaptureState::Incomplete { reason } => {
6010                        StderrCaptureState::Incomplete { reason }
6011                    }
6012                    CaptureState::NotCaptured { reason } => {
6013                        StderrCaptureState::NotCaptured { reason }
6014                    }
6015                },
6016                entries: snapshot
6017                    .entries
6018                    .into_iter()
6019                    .map(|entry| match entry {
6020                        TailEntry::Line {
6021                            text,
6022                            truncated,
6023                            at_ms,
6024                        } => StderrTailEntry::Line {
6025                            text,
6026                            truncated,
6027                            at_ms,
6028                        },
6029                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6030                    })
6031                    .collect(),
6032                dropped_lines: snapshot.dropped_lines,
6033            },
6034        };
6035        Ok(vec![control_response_body_frame(
6036            &frame,
6037            &response,
6038            "ClientControlResponse::SupervisorStderrTail",
6039        )?])
6040    }
6041
6042    async fn observe_control_wait<F: std::future::Future>(
6043        &self,
6044        ctx: &RouteCtx,
6045        target: Option<&str>,
6046        kind: &'static str,
6047        future: F,
6048    ) -> F::Output {
6049        self.counters
6050            .observe_reader_wait(
6051                || {
6052                    let module = self
6053                        .forwarding
6054                        .module_id_for_connection(ctx.connection_id)
6055                        .ok()
6056                        .flatten();
6057                    let principal = if module.is_some() { "module" } else { "direct" };
6058                    (
6059                        self.resolve_reader_wait_target(kind, target)
6060                            .or(module)
6061                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6062                        principal.into(),
6063                    )
6064                },
6065                kind,
6066                future,
6067            )
6068            .await
6069    }
6070
6071    /// Resolve the reader-wait module ID only from supervisor or registry
6072    /// records; never attribute a wait from the request selector alone.
6073    pub(crate) fn resolve_reader_wait_target(
6074        &self,
6075        op: &str,
6076        selector: Option<&str>,
6077    ) -> Option<String> {
6078        let selector = selector?;
6079        match op {
6080            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6081                .supervisor
6082                .get(selector)
6083                .map(|module| module.module_id().to_owned()),
6084            ops::SUPERVISOR_HEALTH_PROBE => self
6085                .registry
6086                .get_module(selector)
6087                .ok()
6088                .flatten()
6089                .map(|registration| registration.manifest.module_id),
6090            _ => None,
6091        }
6092    }
6093
6094    async fn handle_supervisor_terminals(
6095        &self,
6096        ctx: &RouteCtx,
6097        frame: Frame,
6098        module_id: String,
6099    ) -> Result<Vec<Frame>, RouterError> {
6100        let Some(module) = self.supervisor.get(&module_id) else {
6101            return Ok(vec![control_error_frame(
6102                &frame,
6103                "unknown_module",
6104                format!("module_id '{module_id}' is not supervised"),
6105            )?]);
6106        };
6107
6108        // The journal read runs on a blocking thread: it can be megabytes of
6109        // file I/O and must not occupy a runtime worker.
6110        let terminals = self
6111            .observe_control_wait(
6112                ctx,
6113                Some(module.module_id()),
6114                ops::SUPERVISOR_TERMINALS,
6115                module.read_durable_terminal_history(),
6116            )
6117            .await
6118            .map_err(|error| {
6119                RouterError::backend(
6120                    0,
6121                    frame.header.corr,
6122                    format!("failed to read terminal history: {error}"),
6123                )
6124            })?;
6125        let response = ClientControlResponse::SupervisorTerminals {
6126            module_id,
6127            terminals,
6128        };
6129        Ok(vec![control_response_body_frame(
6130            &frame,
6131            &response,
6132            "ClientControlResponse::SupervisorTerminals",
6133        )?])
6134    }
6135
6136    fn handle_supervisor_routes(
6137        &self,
6138        frame: Frame,
6139        module_id: Option<String>,
6140    ) -> Result<Vec<Frame>, RouterError> {
6141        let modules = self
6142            .forwarding
6143            .route_census(module_id.as_deref())
6144            .map_err(RouterError::Forwarding)?
6145            .into_iter()
6146            .map(|(module_id, routes)| SupervisorRouteModule {
6147                module_id,
6148                routes: routes
6149                    .into_iter()
6150                    .map(|route| SupervisorRoute {
6151                        consumer: match route.principal {
6152                            Principal::Reserved { module_id } => {
6153                                SupervisorRouteConsumer::Reserved { module_id }
6154                            }
6155                            Principal::Direct | Principal::Unverified => {
6156                                SupervisorRouteConsumer::Direct {
6157                                    connection_id: route.goodbye_target.connection_id.get(),
6158                                }
6159                            }
6160                        },
6161                        age_ms: Instant::now()
6162                            .saturating_duration_since(route.bound_at)
6163                            .as_millis()
6164                            .try_into()
6165                            .unwrap_or(u64::MAX),
6166                        draining: route.draining,
6167                        drain_reason: route.drain_reason,
6168                    })
6169                    .collect(),
6170            })
6171            .collect();
6172        let response = ClientControlResponse::SupervisorRoutes { modules };
6173        Ok(vec![control_response_body_frame(
6174            &frame,
6175            &response,
6176            "ClientControlResponse::SupervisorRoutes",
6177        )?])
6178    }
6179
6180    async fn handle_supervisor_provenance(
6181        &self,
6182        ctx: &RouteCtx,
6183        frame: Frame,
6184        module_id: Option<String>,
6185    ) -> Result<Vec<Frame>, RouterError> {
6186        let targeted = module_id.is_some();
6187        let selected = if let Some(module_id) = module_id {
6188            let Some(module) = self.supervisor.get(&module_id) else {
6189                return Ok(vec![control_error_frame(
6190                    &frame,
6191                    "unknown_module",
6192                    format!("module_id '{module_id}' is not supervised"),
6193                )?]);
6194            };
6195            vec![module]
6196        } else {
6197            self.supervisor.list()
6198        };
6199
6200        let target = targeted.then(|| selected[0].module_id());
6201        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6202            let mut modules = Vec::with_capacity(selected.len());
6203            for module in &selected {
6204                let (status, observed_image) = module
6205                    .status_and_running_image_agreement()
6206                    .await
6207                    .map_err(|err| {
6208                        RouterError::backend(
6209                            0,
6210                            frame.header.corr,
6211                            format!("failed to read supervisor status: {err}"),
6212                        )
6213                    })?;
6214                let module_declared = self
6215                    .registry
6216                    .get_module(&status.module_id)
6217                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6218                    .and_then(|registration| registration.manifest.provenance)
6219                    .map(|build| ModuleDeclaredProvenance::Reported { build })
6220                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6221                #[cfg(test)]
6222                let running_image = match &self.provenance_probe_override {
6223                    Some(result) => result.clone(),
6224                    None => observed_image,
6225                };
6226                #[cfg(not(test))]
6227                let running_image = observed_image;
6228                modules.push(SupervisorModuleProvenance {
6229                    module_id: status.module_id,
6230                    module_declared,
6231                    daemon_observed: SupervisorObservedProcess {
6232                        launch_nonce_source: status.launch_nonce_source,
6233                        pid: status.pid,
6234                        spawned_at_ms: status.spawned_at_ms,
6235                        spawned_from: status.spawned_from,
6236                        running_image,
6237                    },
6238                });
6239            }
6240            let daemon = SupervisorDaemonProvenance {
6241                daemon_build: self.daemon_provenance.build.clone(),
6242                daemon_observed: DaemonObservedProcess {
6243                    pid: self.daemon_provenance.pid,
6244                    started_at_ms: self
6245                        .daemon_provenance
6246                        .start_clock
6247                        .map(|clock| clock.started_at_ms())
6248                        .or(self.daemon_provenance.started_at_ms),
6249                    running_image: self
6250                        .daemon_provenance
6251                        .probe
6252                        .observe(
6253                            self.daemon_provenance.pid,
6254                            self.daemon_provenance.executable_path.as_deref(),
6255                            self.daemon_provenance.executable_identity.clone(),
6256                            self.daemon_provenance.process_start_time,
6257                        )
6258                        .await,
6259                },
6260            };
6261            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6262            Ok(vec![control_response_body_frame(
6263                &frame,
6264                &response,
6265                "ClientControlResponse::SupervisorProvenance",
6266            )?])
6267        })
6268        .await
6269    }
6270
6271    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6272        self.refresh_capability_requirements();
6273        let generation = self
6274            .registry
6275            .generation()
6276            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6277        let modules = self
6278            .supervisor
6279            .list()
6280            .into_iter()
6281            .map(|module| {
6282                let status = module.status_for_control("health").map_err(|err| {
6283                    RouterError::backend(
6284                        0,
6285                        frame.header.corr,
6286                        format!("failed to read supervisor health: {err}"),
6287                    )
6288                })?;
6289                let module_id = status.module_id;
6290                let capability_detail = self
6291                    .capability_evaluator
6292                    .required_problem_detail(&module_id);
6293                Ok(SupervisorHealthEntry {
6294                    module_id,
6295                    status: status.health.status,
6296                    detail: append_capability_problem_detail(
6297                        status.health.detail,
6298                        capability_detail,
6299                    ),
6300                    metrics: status.health.metrics,
6301                    consecutive_failures: status.health.consecutive_failures,
6302                    late_answer_count: status.health.late_answer_count,
6303                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6304                    last_action: status.health.last_action,
6305                    last_action_ms: status.health.last_action_ms,
6306                    last_probe_ms: status.health.last_probe_ms,
6307                })
6308            })
6309            .collect::<Result<Vec<_>, RouterError>>()?;
6310        let response = ClientControlResponse::SupervisorHealth {
6311            generation,
6312            modules,
6313        };
6314        Ok(vec![control_response_body_frame(
6315            &frame,
6316            &response,
6317            "ClientControlResponse::SupervisorHealth",
6318        )?])
6319    }
6320
6321    async fn handle_supervisor_restart(
6322        &self,
6323        frame: Frame,
6324        module_id: String,
6325        drain_timeout_ms: Option<u64>,
6326        requester: Principal,
6327    ) -> Result<Vec<Frame>, RouterError> {
6328        let operation_lock = self.supervisor.operation_lock();
6329        let _operation_guard = operation_lock.lock().await;
6330        let Some(module) = self.supervisor.get(&module_id) else {
6331            return Ok(vec![control_error_frame(
6332                &frame,
6333                "unknown_module",
6334                format!("module_id '{module_id}' is not supervised"),
6335            )?]);
6336        };
6337
6338        self.route_outages.mark_operator_action(&module_id);
6339        if let Err(err) = module
6340            .restart_requested(drain_timeout_ms, Some(requester))
6341            .await
6342        {
6343            self.route_outages
6344                .operator_action_ended_unrefused(&module_id);
6345            let (code, message) = match err {
6346                crate::supervise::SuperviseError::Disabled { .. } => {
6347                    ("module_disabled", err.to_string())
6348                }
6349                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6350                    ("swap_in_progress", err.to_string())
6351                }
6352                _ => (
6353                    "target_unavailable",
6354                    format!("failed to restart module_id '{module_id}': {err}"),
6355                ),
6356            };
6357            return Ok(vec![control_error_frame(&frame, code, message)?]);
6358        }
6359
6360        let response = ClientControlResponse::SupervisorAck {
6361            module_id,
6362            applied: true,
6363        };
6364        Ok(vec![control_response_body_frame(
6365            &frame,
6366            &response,
6367            "ClientControlResponse::SupervisorAck",
6368        )?])
6369    }
6370
6371    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
6372    /// when the old process has finished draining: a caller whose own lane
6373    /// rides the old process must get its reply before that drain waits on it.
6374    async fn handle_supervisor_swap(
6375        &self,
6376        frame: Frame,
6377        module_id: String,
6378        ready_timeout_ms: Option<u64>,
6379        requester: Principal,
6380    ) -> Result<Vec<Frame>, RouterError> {
6381        // The daemon-wide operation lock is held only to resolve the handle,
6382        // not across the swap. The swap can take its whole readiness budget,
6383        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6384        // holding it here would park an operator's stop behind the swap it is
6385        // meant to abort. A rescan or stop that reaches the module during the
6386        // swap is served by the swap itself (see `supervise_swap`).
6387        let module = {
6388            let operation_lock = self.supervisor.operation_lock();
6389            let _operation_guard = operation_lock.lock().await;
6390            self.supervisor.get(&module_id)
6391        };
6392        let Some(module) = module else {
6393            return Ok(vec![control_error_frame(
6394                &frame,
6395                "unknown_module",
6396                format!("module_id '{module_id}' is not supervised"),
6397            )?]);
6398        };
6399
6400        self.route_outages.mark_operator_action(&module_id);
6401        if let Err(err) = module
6402            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6403            .await
6404        {
6405            self.route_outages
6406                .operator_action_ended_unrefused(&module_id);
6407            use crate::supervise::SuperviseError;
6408            let message = err.to_string();
6409            let error = match err {
6410                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6411                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6412                    code: "swap_refused".to_string(),
6413                    message,
6414                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6415                },
6416                SuperviseError::SwapFailed {
6417                    arm,
6418                    candidate_exit,
6419                    ..
6420                } => ErrorBody {
6421                    code: "swap_failed".to_string(),
6422                    message,
6423                    detail: Some(serde_json::json!({
6424                        "arm": arm.as_str(),
6425                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6426                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6427                    })),
6428                },
6429                _ => ErrorBody::new(
6430                    "target_unavailable",
6431                    format!("failed to swap module_id '{module_id}': {message}"),
6432                ),
6433            };
6434            return Ok(vec![control_error_body_frame(&frame, error)?]);
6435        }
6436        // A completed swap kept the incumbent serving until cutover, so it
6437        // usually opened no outage; a mark left behind would make the next,
6438        // unrelated outage read as requested.
6439        self.route_outages
6440            .operator_action_ended_unrefused(&module_id);
6441
6442        let response = ClientControlResponse::SupervisorAck {
6443            module_id,
6444            applied: true,
6445        };
6446        Ok(vec![control_response_body_frame(
6447            &frame,
6448            &response,
6449            "ClientControlResponse::SupervisorAck",
6450        )?])
6451    }
6452
6453    async fn handle_supervisor_reload(
6454        &self,
6455        frame: Frame,
6456        module_id: String,
6457        requester: Principal,
6458    ) -> Result<Vec<Frame>, RouterError> {
6459        let operation_lock = self.supervisor.operation_lock();
6460        let _operation_guard = operation_lock.lock().await;
6461        let Some(module) = self.supervisor.get(&module_id) else {
6462            return Ok(vec![control_error_frame(
6463                &frame,
6464                "unknown_module",
6465                format!("module_id '{module_id}' is not supervised"),
6466            )?]);
6467        };
6468
6469        self.route_outages.mark_operator_action(&module_id);
6470        if let Err(err) = module.reload_requested(Some(requester)).await {
6471            self.route_outages
6472                .operator_action_ended_unrefused(&module_id);
6473            let (code, message) = match err {
6474                crate::supervise::SuperviseError::Disabled { .. } => {
6475                    ("module_disabled", err.to_string())
6476                }
6477                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6478                    ("swap_in_progress", err.to_string())
6479                }
6480                _ => (
6481                    "reload_failed",
6482                    format!("failed to reload module_id '{module_id}': {err}"),
6483                ),
6484            };
6485            return Ok(vec![control_error_frame(&frame, code, message)?]);
6486        }
6487
6488        let response = ClientControlResponse::SupervisorAck {
6489            module_id,
6490            applied: true,
6491        };
6492        Ok(vec![control_response_body_frame(
6493            &frame,
6494            &response,
6495            "ClientControlResponse::SupervisorAck",
6496        )?])
6497    }
6498
6499    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6500    /// the configuration comes from and whether to apply it; every mode then
6501    /// runs the same comparison, so a preview describes the apply it previews.
6502    async fn handle_supervisor_rescan(
6503        &self,
6504        frame: Frame,
6505        mode: RescanMode,
6506        requester: Principal,
6507    ) -> Result<Vec<Frame>, RouterError> {
6508        let preview = mode.previews();
6509        let Some(context) = self.rescan.clone() else {
6510            return Ok(vec![control_error_frame(
6511                &frame,
6512                "rescan_unavailable",
6513                "the daemon was not started with a reloadable config path".to_string(),
6514            )?]);
6515        };
6516
6517        let operation_lock = self.supervisor.operation_lock();
6518        let _operation_guard = operation_lock.lock().await;
6519        // Loaded under the operation lock, so no other rescan can change the
6520        // staged configuration between a checked apply's comparison and the
6521        // apply itself.
6522        let loaded = match self.load_rescan_config(&mode, &context) {
6523            Ok(config) => config,
6524            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6525        };
6526        // `load` reports a missing file as Ok(None), which is correct at boot
6527        // (no config, nothing to supervise) and catastrophic here: rescan treats
6528        // "not in the config" as "remove it", so an absent file would read as an
6529        // empty module list and retire the entire running fleet. An editor
6530        // writing via write-new-then-rename, or a half-finished edit, is enough
6531        // to open that window. Refuse instead: a config that cannot be read
6532        // carries no instruction to remove anything.
6533        let Some((config, activity_config)) = loaded else {
6534            return Ok(vec![control_error_frame(
6535                &frame,
6536                "invalid_daemon_config",
6537                format!(
6538                    "daemon config not found at {}; refusing to rescan (an absent config would \
6539                     retire every supervised module)",
6540                    context.config_path.display()
6541                ),
6542            )?]);
6543        };
6544        let (
6545            configured_port,
6546            storage_config,
6547            admission_facts_carrier_module_id,
6548            admission_facts_targets,
6549            scope_authority_owners,
6550            privileged_operator_prompt,
6551            modules,
6552            reserved_capabilities,
6553        ) = (
6554            config.port,
6555            config.storage,
6556            config.admission_facts_carrier_module_id,
6557            config.admission_facts_targets,
6558            config.scope_authority_owners,
6559            config.privileged_operator_prompt,
6560            config.modules,
6561            config.reserved_capabilities,
6562        );
6563
6564        // Collect the sections rescan cannot apply, so the REPLY carries them.
6565        //
6566        // The warning below has always been correct and has always gone only to
6567        // the journal -- addressed to whoever reads logs, while the person who
6568        // just edited the config is looking at the CLI. Naming each section
6569        // individually rather than setting a flag: "something outside modules
6570        // changed" sends the operator back to diffing their own file, which is
6571        // the work this is meant to save.
6572        let mut restart_required = Vec::new();
6573        for section in RestartRequiredSection::ALL {
6574            let changed = match section {
6575                RestartRequiredSection::Port => configured_port != context.configured_port,
6576                RestartRequiredSection::Storage => storage_config != context.storage_config,
6577                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6578                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6579                }
6580                RestartRequiredSection::AdmissionFactsTargets => {
6581                    admission_facts_targets != context.admission_facts_targets
6582                }
6583                RestartRequiredSection::ScopeAuthorityOwners => {
6584                    scope_authority_owners != context.scope_authority_owners
6585                }
6586                RestartRequiredSection::PrivilegedOperatorPrompt => {
6587                    privileged_operator_prompt != context.privileged_operator_prompt
6588                }
6589            };
6590            if changed {
6591                restart_required.push(section.label().to_string());
6592            }
6593        }
6594        if !restart_required.is_empty() {
6595            warn!(
6596                config_path = %context.config_path.display(),
6597                sections = %restart_required.join(", "),
6598                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6599            );
6600        }
6601
6602        for configured in &modules {
6603            if let Err(err) = validate_spec(&configured.module_spec()) {
6604                return Ok(vec![control_error_frame(
6605                    &frame,
6606                    "invalid_daemon_config",
6607                    format!("supervisor rescan rejected daemon config: {err}"),
6608                )?]);
6609            }
6610        }
6611
6612        let configured_capabilities = modules
6613            .iter()
6614            .map(|module| (module.module_id.clone(), module.enabled))
6615            .collect::<Vec<_>>();
6616        let preview_capability_warnings = if preview {
6617            let (_, registrations) = self.runtime_capability_snapshot()?;
6618            let current_modules = self
6619                .supervisor
6620                .list()
6621                .into_iter()
6622                .map(|module| module.module_id().to_string())
6623                .collect::<BTreeSet<_>>();
6624            let resulting_modules = configured_capabilities.clone();
6625            let removed = current_modules
6626                .into_iter()
6627                .filter(|module_id| {
6628                    !resulting_modules
6629                        .iter()
6630                        .any(|(configured_id, _)| configured_id == module_id)
6631                })
6632                .collect::<Vec<_>>();
6633            // A candidate preview also names providers the candidate disables.
6634            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6635            let disabled = match &mode {
6636                RescanMode::Candidate(_) => resulting_modules
6637                    .iter()
6638                    .filter(|(_, enabled)| !enabled)
6639                    .map(|(module_id, _)| module_id.clone())
6640                    .collect::<Vec<_>>(),
6641                _ => Vec::new(),
6642            };
6643            self.capability_evaluator.preview_warnings(
6644                resulting_modules,
6645                &removed,
6646                &disabled,
6647                &registrations,
6648            )
6649        } else {
6650            Vec::new()
6651        };
6652        let reconciled = match &mode {
6653            RescanMode::Candidate(candidate) => self
6654                .preview_candidate(&context, modules, candidate)
6655                .map(|result| (result, Vec::new())),
6656            _ => {
6657                self.reconcile_supervised_modules(
6658                    &context.supervisor,
6659                    modules,
6660                    preview,
6661                    requester.clone(),
6662                )
6663                .await
6664            }
6665        };
6666        let (result, changed) = match reconciled {
6667            Ok(result) => result,
6668            Err(message) => {
6669                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6670            }
6671        };
6672        if !preview {
6673            if let Some(ledger) = &self.activity {
6674                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6675                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6676            }
6677            if let Err(code) = self.configure_activity(activity_config).await {
6678                warn!(
6679                    code,
6680                    "activity retention could not be applied during rescan"
6681                );
6682            }
6683            self.capability_evaluator
6684                .configure(configured_capabilities, reserved_capabilities);
6685            self.capability_evaluator.wake_deadline_loop();
6686            self.refresh_capability_requirements();
6687        }
6688        let mut result = result;
6689        result.restart_required = restart_required;
6690        result.capability_warnings = preview_capability_warnings;
6691        let response = ClientControlResponse::SupervisorRescan { result };
6692        Ok(vec![control_response_body_frame(
6693            &frame,
6694            &response,
6695            "ClientControlResponse::SupervisorRescan",
6696        )?])
6697    }
6698
6699    async fn handle_supervisor_release_reserved(
6700        &self,
6701        frame: Frame,
6702        module_id: String,
6703    ) -> Result<Vec<Frame>, RouterError> {
6704        let Some(context) = self.rescan.clone() else {
6705            return Ok(vec![control_error_frame(
6706                &frame,
6707                "release_unavailable",
6708                "reserved-id release requires a daemon started with a reloadable config path",
6709            )?]);
6710        };
6711        let operation_lock = self.supervisor.operation_lock();
6712        let _operation_guard = operation_lock.lock().await;
6713        let loaded = match crate::daemon_config::load(&context.config_path) {
6714            Ok(Some(config)) => config,
6715            Ok(None) => {
6716                return Ok(vec![control_error_frame(
6717                    &frame,
6718                    "invalid_daemon_config",
6719                    format!(
6720                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6721                        context.config_path.display()
6722                    ),
6723                )?])
6724            }
6725            Err(err) => {
6726                return Ok(vec![control_error_frame(
6727                    &frame,
6728                    "invalid_daemon_config",
6729                    format!("unable to verify reserved-id release against daemon config: {err}"),
6730                )?])
6731            }
6732        };
6733        if loaded
6734            .modules
6735            .iter()
6736            .any(|configured| configured.module_id == module_id)
6737        {
6738            return Ok(vec![control_error_frame(
6739                &frame,
6740                "reserved_module_configured",
6741                format!(
6742                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6743                ),
6744            )?]);
6745        }
6746        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6747            return Ok(vec![control_error_frame(
6748                &frame,
6749                "reserved_gate_not_retained",
6750                format!(
6751                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6752                ),
6753            )?]);
6754        }
6755
6756        let response = ClientControlResponse::SupervisorAck {
6757            module_id,
6758            applied: true,
6759        };
6760        Ok(vec![control_response_body_frame(
6761            &frame,
6762            &response,
6763            "ClientControlResponse::SupervisorAck",
6764        )?])
6765    }
6766
6767    /// Compare the configured module set with the supervised one, changing
6768    /// nothing. The staged spec each supervised module holds is what the
6769    /// configured entry is compared with.
6770    fn plan_rescan(
6771        &self,
6772        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6773    ) -> Result<RescanPlan, String> {
6774        let mut current = BTreeMap::new();
6775        for module in self.supervisor.list() {
6776            let (spec, health) = module.configuration().map_err(|err| {
6777                format!(
6778                    "failed to read configuration for module_id '{}': {err}",
6779                    module.module_id()
6780                )
6781            })?;
6782            let enabled = module
6783                .status()
6784                .map_err(|err| {
6785                    format!(
6786                        "failed to read status for module_id '{}': {err}",
6787                        module.module_id()
6788                    )
6789                })?
6790                .enabled;
6791            current.insert(
6792                module.module_id().to_string(),
6793                (module, spec, health, enabled),
6794            );
6795        }
6796        let configured = configured_modules
6797            .into_iter()
6798            .map(|module| (module.module_id.clone(), module))
6799            .collect::<BTreeMap<_, _>>();
6800
6801        let added = configured
6802            .keys()
6803            .filter(|module_id| !current.contains_key(*module_id))
6804            .cloned()
6805            .collect::<Vec<_>>();
6806        let removed = current
6807            .keys()
6808            .filter(|module_id| !configured.contains_key(*module_id))
6809            .cloned()
6810            .collect::<Vec<_>>();
6811        let mut changed_pending_reload = Vec::new();
6812        let mut configuration_changes = BTreeSet::new();
6813        let mut enabled_changes = BTreeSet::new();
6814        let mut unchanged = 0_u32;
6815
6816        for (module_id, configured_module) in &configured {
6817            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6818            else {
6819                continue;
6820            };
6821            // Compare the whole launch spec so a future launch field cannot
6822            // accidentally become a live-only policy change. Health is stored
6823            // separately and applies live without replacing the process.
6824            let launch_changed = *current_spec != configured_module.module_spec();
6825            let configuration_changed =
6826                launch_changed || *current_health != configured_module.health;
6827            let enabled_changed = *current_enabled != configured_module.enabled;
6828            if configuration_changed {
6829                configuration_changes.insert(module_id.clone());
6830            }
6831            if launch_changed {
6832                changed_pending_reload.push(module_id.clone());
6833            }
6834            if enabled_changed {
6835                enabled_changes.insert(module_id.clone());
6836            }
6837            if !configuration_changed && !enabled_changed {
6838                unchanged = unchanged.saturating_add(1);
6839            }
6840        }
6841        Ok(RescanPlan {
6842            current,
6843            configured,
6844            added,
6845            removed,
6846            changed_pending_reload,
6847            configuration_changes,
6848            enabled_changes,
6849            unchanged,
6850        })
6851    }
6852
6853    /// Reconcile the running module set against the configured one.
6854    ///
6855    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6856    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6857    /// deliberately shares this function with the executing path rather than
6858    /// computing the same diff somewhere else -- two implementations of one
6859    /// decision agree until they do not, and the whole value of a preview is that
6860    /// it describes the operation that will actually run.
6861    async fn reconcile_supervised_modules(
6862        &self,
6863        supervisor: &Supervisor,
6864        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6865        preview: bool,
6866        requester: Principal,
6867    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6868        let plan = self.plan_rescan(configured_modules)?;
6869
6870        // Everything above this point is pure computation over two snapshots.
6871        // Everything below MUTATES. The preview returns here so the boundary is a
6872        // single early return rather than a condition repeated at each mutation
6873        // site, where one missed guard would apply part of a change the caller was
6874        // told would not happen.
6875        if preview {
6876            return Ok((plan.preview_result(), Vec::new()));
6877        }
6878        let changed = plan
6879            .configuration_changes
6880            .union(&plan.enabled_changes)
6881            .cloned()
6882            .collect::<Vec<_>>();
6883        let RescanPlan {
6884            current,
6885            configured,
6886            added,
6887            removed,
6888            changed_pending_reload,
6889            configuration_changes,
6890            enabled_changes,
6891            unchanged,
6892        } = plan;
6893        // The staged generation moves before the first change rather than
6894        // after the last, so a rescan that fails partway still invalidates
6895        // every earlier `supervisor.rescan_checked` preview: part of its diff
6896        // may have run.
6897        if !(added.is_empty()
6898            && removed.is_empty()
6899            && configuration_changes.is_empty()
6900            && enabled_changes.is_empty())
6901        {
6902            if let Some(context) = &self.rescan {
6903                context
6904                    .generation
6905                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6906            }
6907        }
6908
6909        for module_id in &removed {
6910            let module = &current
6911                .get(module_id)
6912                .expect("removed module came from current supervisor state")
6913                .0;
6914            module
6915                .retire_requested(Some(requester.clone()))
6916                .await
6917                .map_err(|err| {
6918                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6919                })?;
6920            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6921            //
6922            // `handle_route_open` resolves an absent module in three steps:
6923            // registry, then supervisor status, then tombstone. Retiring first
6924            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6925            // went with the teardown above, the supervisor entry went with
6926            // `retire`, and the tombstone does not exist yet -- so a route.open
6927            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6928            // it") for a module that was deliberately removed and whose caller
6929            // should get `module_removed` (TERMINAL, carrying a removal age).
6930            //
6931            // Writing the tombstone first closes it: during the window the
6932            // supervisor entry still answers, so the caller gets
6933            // `target_unavailable` -- retryable, and TRUE, because the module
6934            // is mid-teardown. After both statements it is `module_removed`.
6935            // No instant remains where a removed module reads as one that
6936            // never existed.
6937            //
6938            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6939            // the absence of a test beside a fix invites deletion: these are
6940            // two sync statements with no await between them, so reaching the
6941            // window needs a second worker thread to land exactly between them
6942            // and there is no hook to force it. MEASURED: the 25 daemon_config
6943            // tests pass identically with the old order and the new one, so
6944            // the existing suite cannot see this and a green run is not
6945            // evidence either way. What the suite does hold is the
6946            // post-condition -- a removed module answers `module_removed` --
6947            // which this preserves.
6948            //
6949            // Found by an Athena panel reading the shipped tree against a
6950            // design note (2026-09-19), as the one concrete instance of that
6951            // note's class that survived contact with source. Direction is
6952            // benign: retryable where terminal was intended, never the reverse.
6953            self.supervisor.record_rescan_removal(module_id);
6954            self.supervisor.retire(module_id);
6955            self.route_outages.forget(module_id);
6956        }
6957
6958        for module_id in configured.keys() {
6959            let Some((module, _, _, _)) = current.get(module_id) else {
6960                continue;
6961            };
6962            let configured_module = configured
6963                .get(module_id)
6964                .expect("configured module id came from configured map");
6965            if configuration_changes.contains(module_id) {
6966                module
6967                    .update_configuration(
6968                        configured_module.module_spec(),
6969                        configured_module.health.clone(),
6970                        configured_module.drain_timeout_ms,
6971                    )
6972                    .await
6973                    .map_err(|err| {
6974                        format!(
6975                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6976                        )
6977                    })?;
6978            }
6979            if enabled_changes.contains(module_id) {
6980                // A rescan that starts or stops a module applies an operator's
6981                // edit to the config, so the resulting outage was asked for.
6982                self.route_outages.mark_operator_action(module_id);
6983                module
6984                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6985                    .await
6986                    .map_err(|err| {
6987                        self.route_outages.operator_action_ended_unrefused(module_id);
6988                        format!(
6989                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6990                            configured_module.enabled
6991                        )
6992                    })?;
6993            }
6994        }
6995
6996        for module_id in &added {
6997            let configured_module = configured
6998                .get(module_id)
6999                .expect("added module id came from configured map");
7000            supervisor
7001                .supervise_configured_requested(
7002                    configured_module.module_spec(),
7003                    configured_module.enabled,
7004                    configured_module.health.clone(),
7005                    configured_module.drain_timeout_ms,
7006                    configured_module.restart,
7007                    configured_module.register_timeout,
7008                    Some(requester.clone()),
7009                )
7010                .map_err(|err| {
7011                    format!("failed to add module_id '{module_id}' during rescan: {err}")
7012                })?;
7013        }
7014
7015        Ok((
7016            SupervisorRescanResult {
7017                added,
7018                removed,
7019                changed_pending_reload,
7020                enabled_changes: enabled_changes.iter().cloned().collect(),
7021                unchanged,
7022                preview: false,
7023                // Filled by the caller, which is the only layer that can see the
7024                // previous config to diff against.
7025                restart_required: Vec::new(),
7026                capability_warnings: Vec::new(),
7027                module_actions: Vec::new(),
7028                earlier_pending: Vec::new(),
7029                expect: None,
7030            },
7031            changed,
7032        ))
7033    }
7034
7035    async fn handle_supervisor_set_enabled(
7036        &self,
7037        frame: Frame,
7038        module_id: String,
7039        enabled: bool,
7040        requester: Principal,
7041    ) -> Result<Vec<Frame>, RouterError> {
7042        let operation_lock = self.supervisor.operation_lock();
7043        let _operation_guard = operation_lock.lock().await;
7044        let Some(module) = self.supervisor.get(&module_id) else {
7045            return Ok(vec![control_error_frame(
7046                &frame,
7047                "unknown_module",
7048                format!("module_id '{module_id}' is not supervised"),
7049            )?]);
7050        };
7051
7052        // Enabling counts as well as disabling: a module an operator starts
7053        // is refused until it registers, and that wait was asked for.
7054        self.route_outages.mark_operator_action(&module_id);
7055        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7056            Ok(applied) => applied,
7057            Err(err) => {
7058                self.route_outages
7059                    .operator_action_ended_unrefused(&module_id);
7060                return Ok(vec![control_error_frame(
7061                    &frame,
7062                    "target_unavailable",
7063                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7064                )?]);
7065            }
7066        };
7067        if !applied {
7068            // Already in the requested state: nothing was made unavailable,
7069            // so the mark must not outlive this request.
7070            self.route_outages
7071                .operator_action_ended_unrefused(&module_id);
7072        }
7073
7074        self.capability_evaluator.wake_deadline_loop();
7075        self.refresh_capability_requirements();
7076        let response = ClientControlResponse::SupervisorAck { module_id, applied };
7077        Ok(vec![control_response_body_frame(
7078            &frame,
7079            &response,
7080            "ClientControlResponse::SupervisorAck",
7081        )?])
7082    }
7083
7084    async fn handle_supervisor_health_probe(
7085        &self,
7086        ctx: &RouteCtx,
7087        frame: Frame,
7088        module_id: String,
7089    ) -> Result<Vec<Frame>, RouterError> {
7090        self.refresh_capability_requirements();
7091        let Some(registration) = self
7092            .registry
7093            .get_module(&module_id)
7094            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7095        else {
7096            return Ok(vec![control_error_frame(
7097                &frame,
7098                "unknown_module",
7099                format!("module_id '{module_id}' is not registered"),
7100            )?]);
7101        };
7102
7103        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
7104        // named for it. Making `module_registration_grants_op` return false
7105        // unconditionally reddens five tests, and every one is named for something
7106        // else -- capability relay, probe/bind demultiplexing, supervision-only
7107        // probing. They exercise a successful advertisement check on the way to their
7108        // own subject.
7109        //
7110        // Real protection, fragile in a specific way: narrowing any of those tests to
7111        // focus on its stated subject would silently remove coverage nobody knows
7112        // they are carrying. Recorded here rather than as a sixth test, because the
7113        // useful fact is WHICH tests hold the guard up -- a new test would add
7114        // coverage without telling the next person what the existing ones quietly do.
7115        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7116        {
7117            return Ok(vec![control_error_frame(
7118                &frame,
7119                "health_not_advertised",
7120                format!("module_id '{module_id}' did not advertise health.check"),
7121            )?]);
7122        }
7123
7124        self.observe_control_wait(
7125            ctx,
7126            Some(&registration.manifest.module_id),
7127            ops::SUPERVISOR_HEALTH_PROBE,
7128            async {
7129                let deadline = Instant::now() + self.health_probe_timeout;
7130                let pending = match self.forwarding.begin_module_control_rpc_for(
7131                    &module_id,
7132                    MODULE_CONTROL_OP_HEALTH_CHECK,
7133                    deadline,
7134                ) {
7135                    Ok(pending) => pending,
7136                    Err(err) => {
7137                        return Ok(vec![control_error_frame(
7138                            &frame,
7139                            forwarding_error_code(&err),
7140                            err.to_string(),
7141                        )?])
7142                    }
7143                };
7144
7145                let PendingModuleControlRpc {
7146                    endpoint,
7147                    module_sink,
7148                    negotiated_ver,
7149                    corr: probe_corr,
7150                    receiver,
7151                } = pending;
7152                let mut guard =
7153                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7154                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7155                    .map_err(|err| {
7156                        RouterError::backend(
7157                            0,
7158                            frame.header.corr,
7159                            format!("failed to encode health.check request: {err}"),
7160                        )
7161                    })?;
7162                let probe_frame = Frame::build_with_version(
7163                    negotiated_ver,
7164                    FrameType::Request,
7165                    control_flags(),
7166                    0,
7167                    0,
7168                    probe_corr,
7169                    probe_body,
7170                )
7171                .map_err(RouterError::FrameBuild)?;
7172
7173                // Queue admission and the reply share one deadline, so a module that
7174                // stops reading cannot park the requesting connection's reader forever.
7175                match timeout_at(deadline, module_sink.send(probe_frame)).await {
7176                    Ok(Ok(())) => {}
7177                    Ok(Err(err)) => {
7178                        return Ok(vec![control_error_frame(
7179                            &frame,
7180                            "target_unavailable",
7181                            err.to_string(),
7182                        )?]);
7183                    }
7184                    Err(_) => {
7185                        return Ok(vec![control_error_frame(
7186                            &frame,
7187                            "module_timeout",
7188                            format!(
7189                                "module_id '{module_id}' did not answer health.check within {:?}",
7190                                self.health_probe_timeout
7191                            ),
7192                        )?]);
7193                    }
7194                }
7195
7196                match timeout_at(deadline, receiver).await {
7197                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7198                        guard.disarm();
7199                        let Some(report) = response.health_report() else {
7200                            return Ok(vec![control_error_frame(
7201                                &frame,
7202                                "invalid_control_body",
7203                                "health.check RPC returned a non-health response",
7204                            )?]);
7205                        };
7206                        // Metrics go out whole here. The supervisor's cached snapshot
7207                        // caps this blob (see truncate_health_metrics), and this path
7208                        // exists precisely to answer without that cap -- so applying it
7209                        // here would leave no way to see what the cached view drops.
7210                        let HealthReport {
7211                            status,
7212                            detail,
7213                            metrics,
7214                        } = report;
7215                        let capability_detail = self
7216                            .capability_evaluator
7217                            .required_problem_detail(&module_id);
7218                        let response = ClientControlResponse::SupervisorHealthProbe {
7219                            module_id,
7220                            status,
7221                            detail: append_capability_problem_detail(detail, capability_detail),
7222                            metrics,
7223                        };
7224                        Ok(vec![control_response_body_frame(
7225                            &frame,
7226                            &response,
7227                            "ClientControlResponse::SupervisorHealthProbe",
7228                        )?])
7229                    }
7230                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7231                        guard.disarm();
7232                        // The module refused the health probe with its own body, so
7233                        // the refusal is relayed without the daemon-origin flag
7234                        // (bit 6): a client must not read it as the daemon's own
7235                        // refusal, which it may treat as never forwarded.
7236                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7237                    }
7238                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7239                        guard.disarm();
7240                        Ok(vec![control_error_frame(
7241                            &frame,
7242                            "target_unavailable",
7243                            message,
7244                        )?])
7245                    }
7246                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7247                        guard.disarm();
7248                        Ok(vec![control_error_frame(
7249                            &frame,
7250                            "invalid_control_body",
7251                            message,
7252                        )?])
7253                    }
7254                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7255                        guard.disarm();
7256                        Ok(vec![control_error_frame(
7257                            &frame,
7258                            "invalid_control_body",
7259                            format!("expected module-control op '{expected}', got '{actual}'"),
7260                        )?])
7261                    }
7262                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7263                        guard.disarm();
7264                        Ok(vec![control_error_frame(
7265                            &frame,
7266                            "module_timeout",
7267                            format!(
7268                                "module_id '{module_id}' answered health.check after {:?}",
7269                                self.health_probe_timeout
7270                            ),
7271                        )?])
7272                    }
7273                    Ok(Err(_)) => Ok(vec![control_error_frame(
7274                        &frame,
7275                        "target_unavailable",
7276                        "health.check waiter was canceled before the module responded",
7277                    )?]),
7278                    Err(_) => Ok(vec![control_error_frame(
7279                        &frame,
7280                        "module_timeout",
7281                        format!(
7282                            "module_id '{module_id}' did not answer health.check within {:?}",
7283                            self.health_probe_timeout
7284                        ),
7285                    )?]),
7286                }
7287            },
7288        )
7289        .await
7290    }
7291
7292    fn supervisor_status(
7293        &self,
7294        module_id: &str,
7295        corr: u64,
7296    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7297        self.supervisor
7298            .get(module_id)
7299            .map(|module| {
7300                let warming = module.is_warming_for_control("status").map_err(|err| {
7301                    RouterError::backend(
7302                        0,
7303                        corr,
7304                        format!(
7305                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
7306                        ),
7307                    )
7308                })?;
7309                module.status_for_control("status").map_err(|err| {
7310                    RouterError::backend(
7311                        0,
7312                        corr,
7313                        format!(
7314                            "failed to read supervisor status for module_id '{module_id}': {err}"
7315                        ),
7316                    )
7317                }).map(|status| (status, warming))
7318            })
7319            .transpose()
7320    }
7321
7322    fn guard_module_control_op(
7323        &self,
7324        frame: &Frame,
7325        module_id: &str,
7326        op: &str,
7327    ) -> Result<Option<Frame>, RouterError> {
7328        if self.module_grants_op(module_id, op, frame.header.corr)? {
7329            return Ok(None);
7330        }
7331
7332        Ok(Some(control_error_frame(
7333            frame,
7334            "op_not_allowed",
7335            format!("module_id '{module_id}' did not grant control op '{op}'"),
7336        )?))
7337    }
7338
7339    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7340        let Some(registration) = self
7341            .registry
7342            .get_module(module_id)
7343            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7344        else {
7345            return Ok(false);
7346        };
7347        Ok(module_registration_grants_op(&registration.control_ops, op))
7348    }
7349
7350    fn handle_status_update(
7351        &self,
7352        endpoint: ModuleEndpointId,
7353        frame: Frame,
7354    ) -> Result<Vec<Frame>, RouterError> {
7355        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7356            Ok(update) => update,
7357            Err(err) => {
7358                // Forward-compat: a newer module may push a channel-0 op this subc
7359                // version doesn't know. The control contract says unknown push ops
7360                // are IGNORED, never answered with an error. Only a malformed body
7361                // for an op we DO know is a real error worth surfacing.
7362                if is_known_module_push_op(&frame.body) {
7363                    return Ok(vec![control_error_frame(
7364                        &frame,
7365                        "invalid_control_body",
7366                        format!("malformed module control push body: {err}"),
7367                    )?]);
7368                }
7369                return Ok(Vec::new());
7370            }
7371        };
7372
7373        match update {
7374            ModuleControlPush::RouteStatus {
7375                route_channel,
7376                route_epoch,
7377                status,
7378            } => {
7379                self.forwarding
7380                    .cache_status(endpoint, route_channel, route_epoch, status)
7381                    .map_err(RouterError::Forwarding)?;
7382            }
7383        }
7384        Ok(Vec::new())
7385    }
7386
7387    fn handle_route_poll(
7388        &self,
7389        ctx: &RouteCtx,
7390        frame: Frame,
7391        route_channel: u16,
7392        route_epoch: u32,
7393        kind: PollKind,
7394    ) -> Result<Vec<Frame>, RouterError> {
7395        let snapshot = self
7396            .forwarding
7397            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7398            .map_err(RouterError::Forwarding)?;
7399        let response = match (kind, snapshot) {
7400            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7401                ClientControlResponse::RoutePoll {
7402                    route_channel,
7403                    route_epoch,
7404                    status,
7405                    live: None,
7406                }
7407            }
7408            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7409                route_channel,
7410                route_epoch,
7411                status: None,
7412                live: None,
7413            },
7414            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7415                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7416                // is what makes reporting `true` correct rather than a
7417                // confident guess. `process_live` returns None only when the
7418                // module id has no supervisor snapshot at all -- an
7419                // externally-started module the daemon did not spawn -- and
7420                // for those the supervisor has no opinion to offer, ever. It
7421                // is never None for a supervised module in an unknown state:
7422                // a supervised module always has a snapshot, and the answer
7423                // comes from `state == Running && process_alive`.
7424                //
7425                // The route is Bound, so the module completed a HELLO on a
7426                // live connection; "the process this route points at is
7427                // running" is therefore attested by the binding rather than
7428                // assumed. Reporting `false` for an unsupervised module would
7429                // be the actual lie -- it would tell a client its healthy
7430                // route is dead because the daemon does not manage the
7431                // process.
7432                //
7433                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7434                // module whose liveness is genuinely unknown, e.g. a snapshot
7435                // that has not been populated yet -- THIS DEFAULT BECOMES
7436                // WRONG and must split: unsupervised stays true, unknown
7437                // becomes null so the client can tell the two apart. The
7438                // response field is already `Option<bool>`, so the wire can
7439                // carry that distinction today.
7440                let live = self
7441                    .process_liveness
7442                    .as_ref()
7443                    .and_then(|source| source.process_live(&module_id))
7444                    .unwrap_or(true);
7445                ClientControlResponse::RoutePoll {
7446                    route_channel,
7447                    route_epoch,
7448                    status: None,
7449                    live: Some(live),
7450                }
7451            }
7452            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7453                route_channel,
7454                route_epoch,
7455                status: None,
7456                live: Some(false),
7457            },
7458        };
7459
7460        Ok(vec![control_response_body_frame(
7461            &frame,
7462            &response,
7463            "ClientControlResponse::RoutePoll",
7464        )?])
7465    }
7466
7467    pub(crate) fn observe_module_control_completion(
7468        &self,
7469        completion: ModuleControlRpcCompletion,
7470    ) -> bool {
7471        match completion {
7472            ModuleControlRpcCompletion::Unknown => false,
7473            ModuleControlRpcCompletion::Settled => true,
7474            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7475                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7476                info!(
7477                    module_id = %module_id,
7478                    latency_ms,
7479                    "late health.check answer proves the module is alive"
7480                );
7481                match self
7482                    .supervisor
7483                    .record_late_health_answer(&module_id, latency_ms)
7484                {
7485                    Ok(true) => {}
7486                    Ok(false) => debug!(
7487                        module_id = %module_id,
7488                        latency_ms,
7489                        "late health.check answer has no active supervisor snapshot"
7490                    ),
7491                    Err(err) => warn!(
7492                        module_id = %module_id,
7493                        latency_ms,
7494                        error = %err,
7495                        "failed to record late health.check answer"
7496                    ),
7497                }
7498                true
7499            }
7500        }
7501    }
7502
7503    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7504    /// the module connection whose frame is being handled, or to the client that
7505    /// relay was opened for.
7506    ///
7507    /// This runs on the MODULE connection's frame handler, where returning `Err`
7508    /// ends that connection -- and a module connection carries every client's
7509    /// routes to that module, so ending it costs the whole fleet its tools.
7510    /// `ConnectionClosing` carries the id of the connection that is closing, and
7511    /// when that id is a CLIENT's, the condition is entirely about that one
7512    /// client's route.open. A client-scoped condition has no authority over a
7513    /// shared module connection, so it is logged and the single relay is dropped:
7514    /// the client is going away, and `complete_pending_relay` already removed the
7515    /// relay before failing, so there is nothing left to settle. Anything that
7516    /// relay still reserved is released by that client's own connection teardown,
7517    /// which is already under way -- that is what "closing" means.
7518    ///
7519    /// Every other failure is a statement about THIS connection and stays fatal:
7520    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7521    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7522    /// frames correctly.
7523    fn refuse_to_end_module_connection_for_a_client(
7524        &self,
7525        module_connection_id: ConnectionId,
7526        corr: u64,
7527        err: ForwardingError,
7528    ) -> Result<(), RouterError> {
7529        if let ForwardingError::ConnectionClosing { connection_id } = err {
7530            if connection_id != module_connection_id {
7531                warn!(
7532                    module_connection_id = module_connection_id.get(),
7533                    client_connection_id = connection_id.get(),
7534                    corr,
7535                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7536                );
7537                return Ok(());
7538            }
7539        }
7540        Err(RouterError::Forwarding(err))
7541    }
7542
7543    fn handle_module_relay_response(
7544        &self,
7545        connection_id: ConnectionId,
7546        frame: Frame,
7547    ) -> Result<Vec<Frame>, RouterError> {
7548        let mut secondary_error = None;
7549        let outcome = match frame.header.ty {
7550            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7551                Ok(probe) if probe.op == "route.bind" => {
7552                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7553                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7554                            RouteBindRelayOutcome::Accepted
7555                        }
7556                        Ok(other) => {
7557                            let message =
7558                                format!("route.bind response carried unexpected body: {other:?}");
7559                            secondary_error = Some(control_error_frame(
7560                                &frame,
7561                                "invalid_control_body",
7562                                message.clone(),
7563                            )?);
7564                            RouteBindRelayOutcome::ModuleGone(message)
7565                        }
7566                        Err(err) => {
7567                            let message = format!("malformed route.bind response body: {err}");
7568                            secondary_error = Some(control_error_frame(
7569                                &frame,
7570                                "invalid_control_body",
7571                                message.clone(),
7572                            )?);
7573                            RouteBindRelayOutcome::ModuleGone(message)
7574                        }
7575                    }
7576                }
7577                Ok(probe) => {
7578                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7579                    {
7580                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7581                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7582                            "malformed {} response body: {err}",
7583                            probe.op
7584                        )),
7585                    };
7586                    let completion = self
7587                        .forwarding
7588                        .complete_module_control_rpc(
7589                            connection_id,
7590                            frame.header.corr,
7591                            Some(&probe.op),
7592                            outcome,
7593                        )
7594                        .map_err(RouterError::Forwarding)?;
7595                    if !self.observe_module_control_completion(completion) {
7596                        debug!(
7597                            connection_id = connection_id.get(),
7598                            corr = frame.header.corr,
7599                            op = %probe.op,
7600                            "dropping late or unknown module-control RPC response"
7601                        );
7602                    }
7603                    return Ok(Vec::new());
7604                }
7605                Err(err) => {
7606                    if let Some(expected_op) = self
7607                        .forwarding
7608                        .pending_module_control_op(connection_id, frame.header.corr)
7609                        .map_err(RouterError::Forwarding)?
7610                    {
7611                        let completion = self
7612                            .forwarding
7613                            .complete_module_control_rpc(
7614                                connection_id,
7615                                frame.header.corr,
7616                                None,
7617                                ModuleControlRpcOutcome::MalformedResponse(format!(
7618                                    "malformed {expected_op} response body: {err}"
7619                                )),
7620                            )
7621                            .map_err(RouterError::Forwarding)?;
7622                        if !self.observe_module_control_completion(completion) {
7623                            debug!(
7624                                connection_id = connection_id.get(),
7625                                corr = frame.header.corr,
7626                                "dropping late malformed module-control RPC response"
7627                            );
7628                        }
7629                        return Ok(Vec::new());
7630                    }
7631                    let message = format!("malformed route.bind response body: {err}");
7632                    secondary_error = Some(control_error_frame(
7633                        &frame,
7634                        "invalid_control_body",
7635                        message.clone(),
7636                    )?);
7637                    RouteBindRelayOutcome::ModuleGone(message)
7638                }
7639            },
7640            FrameType::Error => {
7641                if self
7642                    .forwarding
7643                    .pending_module_control_op(connection_id, frame.header.corr)
7644                    .map_err(RouterError::Forwarding)?
7645                    .is_some()
7646                {
7647                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7648                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7649                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7650                            "malformed module-control ERROR body: {err}"
7651                        )),
7652                    };
7653                    let completion = self
7654                        .forwarding
7655                        .complete_module_control_rpc(
7656                            connection_id,
7657                            frame.header.corr,
7658                            None,
7659                            outcome,
7660                        )
7661                        .map_err(RouterError::Forwarding)?;
7662                    if !self.observe_module_control_completion(completion) {
7663                        debug!(
7664                            connection_id = connection_id.get(),
7665                            corr = frame.header.corr,
7666                            "dropping late or unknown module-control RPC error"
7667                        );
7668                    }
7669                    return Ok(Vec::new());
7670                }
7671                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7672                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7673                    Err(err) => {
7674                        let message = format!("malformed route.bind ERROR body: {err}");
7675                        secondary_error = Some(control_error_frame(
7676                            &frame,
7677                            "invalid_control_body",
7678                            message.clone(),
7679                        )?);
7680                        RouteBindRelayOutcome::ModuleGone(message)
7681                    }
7682                }
7683            }
7684            ty => {
7685                return Ok(vec![control_error_frame(
7686                    &frame,
7687                    "unsupported_control_frame",
7688                    format!("unsupported module channel-0 frame {ty:?}"),
7689                )?])
7690            }
7691        };
7692
7693        let settled =
7694            self.forwarding
7695                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7696        let completion = match settled {
7697            Ok(completion) => completion,
7698            Err(err) => {
7699                self.refuse_to_end_module_connection_for_a_client(
7700                    connection_id,
7701                    frame.header.corr,
7702                    err,
7703                )?;
7704                return Ok(secondary_error.into_iter().collect());
7705            }
7706        };
7707        if let Some(target) = completion.abandoned.as_ref() {
7708            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7709        }
7710        if !completion.settled {
7711            debug!(
7712                connection_id = connection_id.get(),
7713                corr = frame.header.corr,
7714                frame_type = ?frame.header.ty,
7715                "dropping late or unknown route.bind relay response"
7716            );
7717        }
7718        Ok(secondary_error.into_iter().collect())
7719    }
7720
7721    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7722        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7723        // GOODBYE ends the connection's logical session even when its socket
7724        // stays open. Use disconnect teardown so verdicts, client notices and
7725        // scope authority are released at the same lifecycle boundary.
7726        self.cleanup_connection_with_end_reason(
7727            connection_id,
7728            RegistrationEndReason::ExplicitGoodbye,
7729        )
7730        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7731        Ok(Vec::new())
7732    }
7733}
7734
7735impl Default for ControlHandler {
7736    fn default() -> Self {
7737        Self::new(Arc::new(Registry::default()))
7738    }
7739}
7740
7741impl crate::supervise::SwapPromotionObserver for ControlHandler {
7742    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7743        self.apply_registration_capabilities(registration);
7744    }
7745}
7746
7747fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7748    CapabilityRequirementStatus {
7749        consumer: status.consumer,
7750        capability: status.capability,
7751        need: match status.need {
7752            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7753            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7754        },
7755        verdict: status.verdict.as_str().to_string(),
7756        episode_seq: status.episode_seq,
7757        config_satisfiable: status.config_satisfiable,
7758        runtime_available: status.runtime_available,
7759        detail: status.detail,
7760    }
7761}
7762
7763fn append_capability_problem_detail(
7764    detail: Option<String>,
7765    capability_detail: Option<String>,
7766) -> Option<String> {
7767    match (detail, capability_detail) {
7768        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7769        (Some(detail), None) => Some(detail),
7770        (None, Some(capability_detail)) => Some(capability_detail),
7771        (None, None) => None,
7772    }
7773}
7774
7775fn subc_ops() -> Vec<String> {
7776    SUBC_CONTROL_OPS
7777        .iter()
7778        .map(|op| (*op).to_string())
7779        .collect()
7780}
7781
7782fn module_subc_ops() -> Vec<String> {
7783    SUBC_CONTROL_OPS
7784        .iter()
7785        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7786        .map(|op| (*op).to_string())
7787        .collect()
7788}
7789
7790#[cfg(test)]
7791fn module_baseline_control_ops() -> Vec<String> {
7792    MODULE_BASELINE_CONTROL_OPS
7793        .iter()
7794        .map(|op| (*op).to_string())
7795        .collect()
7796}
7797
7798fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7799    let mut seen = HashSet::new();
7800    let mut effective = Vec::new();
7801    for op in MODULE_BASELINE_CONTROL_OPS {
7802        if seen.insert((*op).to_string()) {
7803            effective.push((*op).to_string());
7804        }
7805    }
7806    for op in declared.unwrap_or_default() {
7807        if seen.insert(op.clone()) {
7808            effective.push(op);
7809        }
7810    }
7811    effective
7812}
7813
7814fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7815    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7816}
7817
7818fn target_module_id(target: &RouteTarget) -> &str {
7819    match target {
7820        RouteTarget::ToolProvider { module_id }
7821        | RouteTarget::ManagementSurface { module_id }
7822        | RouteTarget::InternalService { module_id, .. } => module_id,
7823    }
7824}
7825
7826fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7827    roles.iter().any(|role| match (target, role) {
7828        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7829        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7830        (
7831            RouteTarget::InternalService { service_id, .. },
7832            ProviderRole::InternalService {
7833                service_id: provided,
7834                ..
7835            },
7836        ) => service_id == provided,
7837        _ => false,
7838    })
7839}
7840
7841fn is_routable_role(role: &ProviderRole) -> bool {
7842    matches!(
7843        role,
7844        ProviderRole::ToolProvider { .. }
7845            | ProviderRole::ManagementSurface { .. }
7846            | ProviderRole::InternalService { .. }
7847    )
7848}
7849
7850#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7851enum ControlRequestBodyError {
7852    UnknownOp,
7853    InvalidBody,
7854}
7855
7856#[derive(Debug, Deserialize)]
7857struct ControlOpProbe {
7858    op: String,
7859}
7860
7861/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7862/// this set is treated as a forward-compat unknown and ignored rather than errored.
7863const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7864
7865fn is_known_module_push_op(body: &[u8]) -> bool {
7866    serde_json::from_slice::<ControlOpProbe>(body)
7867        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7868        .unwrap_or(false)
7869}
7870
7871fn is_known_module_request_op(body: &[u8]) -> bool {
7872    serde_json::from_slice::<ControlOpProbe>(body)
7873        .map(|probe| is_module_to_subc_op(&probe.op))
7874        .unwrap_or(false)
7875}
7876
7877fn is_module_to_subc_op(op: &str) -> bool {
7878    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7879}
7880
7881fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7882    debug!(
7883        op = %op,
7884        connection_id = connection_id.get(),
7885        corr,
7886        "control dispatch"
7887    );
7888}
7889
7890fn log_slow_control_dispatch(
7891    dispatch_started_at: Option<StdInstant>,
7892    op: &'static str,
7893    connection_id: ConnectionId,
7894    corr: u64,
7895    route_open_timings: Option<&RouteOpenTimings>,
7896) {
7897    let Some(dispatch_started_at) = dispatch_started_at else {
7898        return;
7899    };
7900    // Use the same monotonic clock as the phase timers and relay deadline.
7901    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7902    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7903        if let Some(timings) = route_open_timings {
7904            warn!(
7905                op = %op,
7906                connection_id = connection_id.get(),
7907                corr,
7908                elapsed_ms = elapsed.as_millis() as u64,
7909                module_id = ?timings.module_id,
7910                principal = %timings.principal,
7911                admission_ms = timings.admission.as_millis() as u64,
7912                project_root_ms = timings.project_root.as_millis() as u64,
7913                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7914                bind_send_ms = timings.bind_send.as_millis() as u64,
7915                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7916                "slow control dispatch"
7917            );
7918            return;
7919        }
7920        warn!(
7921            op = %op,
7922            connection_id = connection_id.get(),
7923            corr,
7924            elapsed_ms = elapsed.as_millis() as u64,
7925            "slow control dispatch"
7926        );
7927    }
7928}
7929
7930/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7931#[cfg(test)]
7932type TestCallerLookup =
7933    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7934
7935/// Every new client operation needs an explicit privilege decision here.
7936fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7937    match request {
7938        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7939            Some(PrivilegedOperation::Restart { module_id })
7940        }
7941        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7942            Some(PrivilegedOperation::Swap { module_id })
7943        }
7944        ClientControlRequest::SupervisorReload { module_id } => {
7945            Some(PrivilegedOperation::Reload { module_id })
7946        }
7947        ClientControlRequest::SupervisorRescan { preview } => {
7948            if *preview {
7949                None
7950            } else {
7951                Some(PrivilegedOperation::Rescan)
7952            }
7953        }
7954        // Only the well-formed apply asks for approval. A preview changes
7955        // nothing, and any other combination is refused before it touches
7956        // anything, so prompting the operator for it would ask about a request
7957        // that cannot run.
7958        ClientControlRequest::SupervisorRescanChecked {
7959            preview,
7960            candidate,
7961            expect,
7962        } => (!*preview && candidate.is_none() && expect.is_some())
7963            .then_some(PrivilegedOperation::Rescan),
7964        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7965            Some(PrivilegedOperation::SetEnabled {
7966                module_id,
7967                enabled: *enabled,
7968            })
7969        }
7970        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7971            Some(PrivilegedOperation::ReleaseReserved { module_id })
7972        }
7973        ClientControlRequest::ServerDescribe {}
7974        // The announcement uses the same client admission as supervisor.restart,
7975        // but cannot stop anything itself and needs no operator confirmation.
7976        | ClientControlRequest::DaemonShutdownIntent { .. }
7977        | ClientControlRequest::CatalogList { .. }
7978        | ClientControlRequest::RouteOpen { .. }
7979        | ClientControlRequest::RoutePoll { .. }
7980        | ClientControlRequest::SupervisorList {}
7981        | ClientControlRequest::SupervisorHealth {}
7982        | ClientControlRequest::SupervisorHealthProbe { .. }
7983        | ClientControlRequest::SupervisorRoutes { .. }
7984        | ClientControlRequest::SupervisorProvenance { .. }
7985        | ClientControlRequest::SupervisorStderrTail { .. }
7986        | ClientControlRequest::SupervisorTerminals { .. }
7987        | ClientControlRequest::SupervisorSpawnSnapshot {}
7988        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7989    }
7990}
7991
7992fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7993    match request {
7994        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
7995        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7996        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7997        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7998        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7999        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8000        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8001        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8002        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8003        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8004        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8005        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8006        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8007        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8008        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8009        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8010        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8011        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8012        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8013        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8014        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8015    }
8016}
8017
8018/// The caller of one `route.open`, as its `route.open refused` line names it.
8019///
8020/// A connection id alone cannot name a caller after the fact: ids restart at
8021/// every daemon restart, so the same number belongs to different modules
8022/// before and after one. These three fields are what the refused line needs
8023/// to be attributable on its own, and they are rendered with the same helpers
8024/// as the `route.open accepted` line so both directions grep alike.
8025struct RouteOpenCaller {
8026    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
8027    /// when the request presented a consumer identity whose launch nonce did
8028    /// not verify. The claimed module id is never printed as a principal.
8029    principal: String,
8030    /// `none` for an unscoped open; the admitted stamp's label once scope
8031    /// admission issued one; before that, the requested selector's label.
8032    scope: String,
8033    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
8034    /// because it is a string the caller chose.
8035    harness: String,
8036}
8037
8038impl RouteOpenCaller {
8039    fn new(
8040        principal: Option<&Principal>,
8041        scope: Option<&ScopeSelector>,
8042        identity: &BindIdentity,
8043    ) -> Self {
8044        Self {
8045            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8046            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8047            harness: bounded_log_text(&identity.harness),
8048        }
8049    }
8050
8051    /// For a refusal that has no parsed request at hand: names whatever the
8052    /// frame still says, and `unknown` where it says nothing parseable.
8053    fn unknown() -> Self {
8054        Self {
8055            principal: "unknown".to_string(),
8056            scope: "unknown".to_string(),
8057            harness: "unknown".to_string(),
8058        }
8059    }
8060}
8061
8062/// Longest client-chosen text (a scope ref, a harness) a route.open log line
8063/// prints before cutting it.
8064const MAX_LOGGED_TEXT_CHARS: usize = 64;
8065
8066/// `value` Debug-quoted, so control characters land escaped and cannot forge
8067/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
8068/// `…` inside the quotes so one request cannot flood the line.
8069fn bounded_log_text(value: &str) -> String {
8070    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8071    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8072        bounded.push('…');
8073    }
8074    format!("{bounded:?}")
8075}
8076
8077/// The `scope=` field of the `route.open accepted` line, from the stamp that
8078/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
8079/// so a ref holding control characters cannot forge a log line, and cut to 64
8080/// characters with a trailing `…` to bound the line. The bind proof and the
8081/// scope's attributes never appear.
8082fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8083    use subc_protocol::scope::ScopeKind;
8084    let kind = match stamp.kind {
8085        ScopeKind::Head => "head",
8086        ScopeKind::Worker => "worker",
8087        ScopeKind::Ephemeral => "ephemeral",
8088    };
8089    format!(
8090        "{kind}/{}/{}@{}",
8091        principal_label(&stamp.owner),
8092        bounded_log_text(&stamp.scope_ref),
8093        stamp.scope_epoch
8094    )
8095}
8096
8097/// The `scope=` field of a `route.open refused` line for an open refused
8098/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
8099/// The kind is not known yet, so the literal `requested` stands in its place.
8100/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
8101/// owner it has not been checked against the configured modules: a quoted
8102/// owner means the value was never validated. The bind proof never appears.
8103fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8104    let epoch = selector
8105        .scope_epoch
8106        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8107    format!(
8108        "requested/{}/{}@{epoch}",
8109        bounded_log_text(&principal_label(&selector.owner)),
8110        bounded_log_text(&selector.scope_ref),
8111    )
8112}
8113
8114fn principal_label(principal: &Principal) -> String {
8115    match principal {
8116        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8117        Principal::Direct => "direct".to_string(),
8118        other => format!("{other:?}"),
8119    }
8120}
8121
8122fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8123    match request {
8124        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8125        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8126        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8127        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8128        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8129    }
8130}
8131
8132fn parse_client_control_request(
8133    body: &[u8],
8134) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8135    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8136        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8137            Ok(probe)
8138                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8139                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8140            {
8141                ControlRequestBodyError::InvalidBody
8142            }
8143            Ok(_) => ControlRequestBodyError::UnknownOp,
8144            Err(_) => ControlRequestBodyError::InvalidBody,
8145        };
8146        (err, classification)
8147    })
8148}
8149
8150fn parse_module_control_request_from_module(
8151    body: &[u8],
8152) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8153    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8154        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8155            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8156            Ok(_) => ControlRequestBodyError::UnknownOp,
8157            Err(_) => ControlRequestBodyError::InvalidBody,
8158        };
8159        (err, classification)
8160    })
8161}
8162
8163#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8164enum ProviderRoleKind {
8165    ToolProvider,
8166    PipelineStage,
8167    ManagementSurface,
8168    InternalService,
8169}
8170
8171fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8172    match role {
8173        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8174        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8175        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8176        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8177    }
8178}
8179
8180fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8181    roles.iter().map(provider_role_kind).collect()
8182}
8183
8184/// Most refused scope records named individually in the log per sync; the
8185/// `refused` count on the accepted line is always complete.
8186const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8187
8188/// Per-outcome counts of one accepted `scope.sync`, for its log line.
8189#[derive(Debug, Default, PartialEq, Eq)]
8190struct ScopeOutcomeCounts {
8191    created: usize,
8192    replaced: usize,
8193    updated: usize,
8194    unchanged: usize,
8195    refused: usize,
8196}
8197
8198impl ScopeOutcomeCounts {
8199    fn of(results: &[ScopeRecordResult]) -> Self {
8200        let mut counts = Self::default();
8201        for result in results {
8202            let slot = match result.outcome {
8203                ScopeRecordOutcome::Created => &mut counts.created,
8204                ScopeRecordOutcome::Replaced => &mut counts.replaced,
8205                ScopeRecordOutcome::Updated => &mut counts.updated,
8206                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8207                ScopeRecordOutcome::Refused => &mut counts.refused,
8208            };
8209            *slot += 1;
8210        }
8211        counts
8212    }
8213}
8214
8215#[cfg(test)]
8216mod scope_outcome_count_tests {
8217    use super::*;
8218
8219    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8220        ScopeRecordResult {
8221            scope_ref: "r".to_string(),
8222            scope_epoch: 1,
8223            outcome,
8224            code: None,
8225            message: None,
8226            version: None,
8227            parent_state: None,
8228        }
8229    }
8230
8231    /// Each outcome lands in its own count, so a refused record can never be
8232    /// hidden inside the total the log already printed.
8233    #[test]
8234    fn every_outcome_is_counted_in_its_own_field() {
8235        let results = [
8236            result(ScopeRecordOutcome::Created),
8237            result(ScopeRecordOutcome::Created),
8238            result(ScopeRecordOutcome::Replaced),
8239            result(ScopeRecordOutcome::Updated),
8240            result(ScopeRecordOutcome::Unchanged),
8241            result(ScopeRecordOutcome::Refused),
8242            result(ScopeRecordOutcome::Refused),
8243            result(ScopeRecordOutcome::Refused),
8244        ];
8245        assert_eq!(
8246            ScopeOutcomeCounts::of(&results),
8247            ScopeOutcomeCounts {
8248                created: 2,
8249                replaced: 1,
8250                updated: 1,
8251                unchanged: 1,
8252                refused: 3,
8253            }
8254        );
8255    }
8256}
8257
8258/// Return whether a catalog change can create a newly violating live route.
8259/// Removing an attested claim is intentionally excluded: it makes fewer routes
8260/// forbidden and therefore must leave the existing route census untouched.
8261fn capability_census_trigger(
8262    old: Option<&CapabilityDeclarations>,
8263    new: Option<&CapabilityDeclarations>,
8264) -> bool {
8265    let old_provides = old
8266        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8267        .unwrap_or_default();
8268    let old_denies = old
8269        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8270        .unwrap_or_default();
8271    let new = new.cloned().unwrap_or(CapabilityDeclarations {
8272        provides: Vec::new(),
8273        requires: Vec::new(),
8274        must_never_reach: Vec::new(),
8275    });
8276
8277    new.provides
8278        .iter()
8279        .any(|capability| !old_provides.contains(capability))
8280        || new
8281            .must_never_reach
8282            .iter()
8283            .any(|capability| !old_denies.contains(capability))
8284}
8285
8286/// Find the first capability an attested opener denies that an attested target
8287/// claims. Both manifests are live registry records, never cached or client data.
8288fn denied_capability<'a>(
8289    opening_manifest: &'a ModuleManifest,
8290    target_manifest: &ModuleManifest,
8291) -> Option<&'a str> {
8292    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8293    let target_capabilities = target_manifest.capabilities.as_ref()?;
8294    opening_capabilities
8295        .must_never_reach
8296        .iter()
8297        .find(|denied| {
8298            target_capabilities
8299                .provides
8300                .iter()
8301                .any(|provided| provided == *denied)
8302        })
8303        .map(String::as_str)
8304}
8305
8306fn catalog_update_frozen_field_message(
8307    registered: &ModuleManifest,
8308    provides: &[ProviderRole],
8309) -> Option<String> {
8310    let old_has_provides = !registered.provides.is_empty();
8311    let new_has_provides = !provides.is_empty();
8312    if old_has_provides != new_has_provides {
8313        return Some(format!(
8314            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8315            registered.module_id
8316        ));
8317    }
8318
8319    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
8320        return Some(format!(
8321            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8322            registered.module_id
8323        ));
8324    }
8325
8326    let registered_concurrency = manifest_concurrency(registered);
8327    let mut candidate = registered.clone();
8328    candidate.provides = provides.to_vec();
8329    let candidate_concurrency = manifest_concurrency(&candidate);
8330    if candidate_concurrency != registered_concurrency {
8331        return Some(format!(
8332            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8333            registered.module_id, registered_concurrency, candidate_concurrency
8334        ));
8335    }
8336
8337    // control_ops live beside the manifest in the HELLO body, not inside
8338    // ModuleManifest, so a provides-only catalog.update cannot change them.
8339    None
8340}
8341
8342fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8343    manifest.provides.iter().any(is_routable_role)
8344}
8345
8346/// Returns the routable-provider concurrency subc should enforce for this manifest.
8347///
8348/// ToolProvider and ManagementSurface store their delivery concurrency directly.
8349/// InternalService has no role-specific concurrency field, so it retains the
8350/// existing ModuleManaged default for backward compatibility.
8351fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8352    manifest
8353        .provides
8354        .iter()
8355        .find_map(|provider| match provider {
8356            ProviderRole::ToolProvider { concurrency, .. }
8357            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8358            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8359        })
8360        .unwrap_or(Concurrency::ModuleManaged)
8361}
8362
8363/// True when the manifest carries a ManagementSurface role whose concurrency
8364/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
8365/// bytes because the typed manifest deliberately erases that distinction: the
8366/// default exists for wire compatibility, and this probe exists so the default
8367/// stays observable. Any parse irregularity returns false -- the caller only
8368/// logs, and a malformed body already failed registration upstream.
8369fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8370    let has_management_surface = manifest
8371        .provides
8372        .iter()
8373        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8374    if !has_management_surface {
8375        return false;
8376    }
8377    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8378        return false;
8379    };
8380    let Some(provides) = raw
8381        .get("manifest")
8382        .and_then(|manifest| manifest.get("provides"))
8383        .and_then(serde_json::Value::as_array)
8384    else {
8385        return false;
8386    };
8387    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
8388    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
8389    // against the management_surface_manifest_without_concurrency golden, not
8390    // recalled (the externally-tagged guess was this function's first bug).
8391    provides.iter().any(|role| {
8392        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8393            && role.get("concurrency").is_none()
8394    })
8395}
8396
8397fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8398    if peer_version != PROTOCOL_VERSION {
8399        return Err(format!(
8400            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8401        ));
8402    }
8403    Ok(PROTOCOL_VERSION)
8404}
8405
8406fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8407    Frame::build_with_version(
8408        response_version(frame),
8409        FrameType::Pong,
8410        frame.header.flags,
8411        0,
8412        0,
8413        frame.header.corr,
8414        Vec::new(),
8415    )
8416    .map_err(RouterError::FrameBuild)
8417}
8418
8419pub(crate) fn control_error_frame(
8420    frame: &Frame,
8421    code: &'static str,
8422    message: impl Into<String>,
8423) -> Result<Frame, RouterError> {
8424    control_error_body_frame(
8425        frame,
8426        ErrorBody {
8427            code: code.to_string(),
8428            message: message.into(),
8429            detail: None,
8430        },
8431    )
8432}
8433
8434/// A channel-0 ERROR for a refusal the daemon decided itself: every
8435/// route.open refusal, the drain `module_reloading` and scope-change/end
8436/// bodies, spawn-subscription and supervisor refusals. Carries the
8437/// daemon-origin bit, which is what lets a client treat `module_reloading`
8438/// as proof that its request never reached a module. Response, push,
8439/// goodbye and relay-request frames keep the plain control flags in their
8440/// own builders.
8441fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8442    control_error_frame_with_flags(frame, error, daemon_error_flags())
8443}
8444
8445/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8446/// refused route.bind answered as a route.open refusal, or a refused
8447/// module-control RPC. Plain control flags — the code and message are
8448/// module-chosen, so the frame must not claim daemon authorship, even when
8449/// the module picks a daemon code like `module_reloading`.
8450fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8451    control_error_frame_with_flags(frame, error, control_flags())
8452}
8453
8454fn control_error_frame_with_flags(
8455    frame: &Frame,
8456    error: ErrorBody,
8457    flags: Flags,
8458) -> Result<Frame, RouterError> {
8459    let body = serde_json::to_vec(&error).map_err(|err| {
8460        RouterError::backend(
8461            0,
8462            frame.header.corr,
8463            format!("failed to encode control ERROR: {err}"),
8464        )
8465    })?;
8466
8467    Frame::build_with_version(
8468        response_version(frame),
8469        FrameType::Error,
8470        flags,
8471        0,
8472        0,
8473        frame.header.corr,
8474        body,
8475    )
8476    .map_err(RouterError::FrameBuild)
8477}
8478
8479pub(crate) fn control_response_body_frame<T: Serialize>(
8480    frame: &Frame,
8481    reply: &T,
8482    label: &'static str,
8483) -> Result<Frame, RouterError> {
8484    let body = serde_json::to_vec(reply).map_err(|err| {
8485        RouterError::backend(
8486            0,
8487            frame.header.corr,
8488            format!("failed to encode {label}: {err}"),
8489        )
8490    })?;
8491
8492    Frame::build_with_version(
8493        response_version(frame),
8494        FrameType::Response,
8495        control_flags(),
8496        0,
8497        0,
8498        frame.header.corr,
8499        body,
8500    )
8501    .map_err(RouterError::FrameBuild)
8502}
8503
8504/// Map a forwarding failure to the wire code a client sees.
8505///
8506/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8507/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8508/// chosen here decides whether a caller retries or gives up.
8509///
8510/// That makes attribution the load-bearing property, not merely having a code. A
8511/// permanent fault published as a retryable one produces a fleet-wide retry storm
8512/// against something that can never recover; a transient fault published as
8513/// permanent gives up on work that would have succeeded. Both look correct in a
8514/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8515/// the mapping per variant rather than merely asserting that some code exists.
8516///
8517/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8518/// two codes on the same side of the boundary passes it. Measured rather than
8519/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8520/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8521/// a test named for something else that happens to assert the string.
8522///
8523/// That accidental coverage is deliberately left alone rather than promoted to a
8524/// named test, because it guards a property this function does not promise.
8525/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8526/// specific code within a class, so identity is free to change and only the
8527/// partition is a contract. Splitting it out would assert a guarantee nothing
8528/// depends on — and a suite that promises more than the code does is the harder
8529/// thing to correct later, because the next reader cannot tell which assertions
8530/// are load-bearing.
8531///
8532/// Pin identity here the moment a consumer branches on a specific code.
8533fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8534    match err {
8535        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8536        ForwardingError::NoModuleConnection => "target_unavailable",
8537        ForwardingError::ModuleReloading { .. } => "module_reloading",
8538        ForwardingError::ClientRouteChannelExhausted { .. }
8539        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8540        ForwardingError::StaleModuleEndpoint
8541        | ForwardingError::UnknownReservation { .. }
8542        | ForwardingError::ConnectionClosing { .. }
8543        | ForwardingError::ClientEgressClosed { .. }
8544        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8545        // Only a swap candidate's registration can produce this, and it means
8546        // exactly what a second active HELLO for a live id means.
8547        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8548        ForwardingError::RelayCorrelationExhausted
8549        | ForwardingError::RouteOpenBuild(_)
8550        | ForwardingError::Poisoned => "forwarding_error",
8551    }
8552}
8553
8554fn response_version(frame: &Frame) -> u8 {
8555    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8556        frame.header.ver
8557    } else {
8558        PROTOCOL_VERSION
8559    }
8560}
8561
8562fn control_flags() -> Flags {
8563    Flags::new(false, Priority::Passive, false)
8564}
8565
8566/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8567/// channel. The target is the module (a client never saw the route), so this
8568/// takes the module path: delivered late rather than dropped when the module's
8569/// queue is momentarily full, and never closing its connection.
8570fn send_goodbye_target_best_effort(
8571    counters: &DaemonCounters,
8572    target: &GoodbyeTarget,
8573    context: &'static str,
8574) {
8575    let Ok(frame) = Frame::build_with_version(
8576        target.negotiated_ver,
8577        FrameType::Goodbye,
8578        control_flags(),
8579        target.channel,
8580        target.epoch,
8581        0,
8582        Vec::new(),
8583    ) else {
8584        return;
8585    };
8586    crate::forwarding::send_module_route_goodbye(
8587        counters,
8588        &target.sink,
8589        frame,
8590        target.module_id.as_deref(),
8591        context,
8592    );
8593}
8594
8595pub(crate) fn send_route_control_pushes(
8596    forwarding: &ForwardingTable,
8597    routes: Vec<EndpointRoute>,
8598    push: ClientControlPush,
8599) {
8600    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8601    for route in routes {
8602        let target = route.goodbye_target;
8603        if let Some((existing, channels)) = targets
8604            .iter_mut()
8605            .find(|(existing, _)| existing.connection_id == target.connection_id)
8606        {
8607            debug_assert_eq!(
8608                existing.negotiated_ver, target.negotiated_ver,
8609                "one connection cannot negotiate multiple frame versions"
8610            );
8611            if !channels.contains(&target.channel) {
8612                channels.push(target.channel);
8613            }
8614            continue;
8615        }
8616        let channel = target.channel;
8617        targets.push((target, vec![channel]));
8618    }
8619    for (target, mut channels) in targets {
8620        channels.sort_unstable();
8621        let mut push = push.clone();
8622        match &mut push {
8623            ClientControlPush::RouteClosing {
8624                channels: covered, ..
8625            }
8626            | ClientControlPush::RouteClosed {
8627                channels: covered, ..
8628            } => *covered = channels,
8629        }
8630        let body = match serde_json::to_vec(&push) {
8631            Ok(body) => body,
8632            Err(err) => {
8633                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8634                continue;
8635            }
8636        };
8637        let frame = match Frame::build_with_version(
8638            target.negotiated_ver,
8639            FrameType::Push,
8640            control_flags(),
8641            0,
8642            0,
8643            0,
8644            body.clone(),
8645        ) {
8646            Ok(frame) => frame,
8647            Err(err) => {
8648                warn!(
8649                    route_channel = target.channel,
8650                    error = %err,
8651                    "failed to build route lifecycle control PUSH frame"
8652                );
8653                continue;
8654            }
8655        };
8656        if let Err(err) = target.sink.try_send(frame) {
8657            if target.close_on_delivery_failure() {
8658                warn!(
8659                    target_connection_id = target.connection_id.get(),
8660                    route_channel = target.channel,
8661                    error = %err,
8662                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8663                );
8664                let _ = forwarding.escalate_client_delivery_failure(
8665                    target.connection_id,
8666                    target.channel,
8667                    target.epoch,
8668                    CloseReason::new(
8669                        "route_lifecycle_push_delivery_failed",
8670                        format!(
8671                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8672                            target.channel
8673                        ),
8674                    ),
8675                    crate::forwarding::UndeliveredFrame {
8676                        module_id: target.module_id.as_deref(),
8677                        sink: &target.sink,
8678                    },
8679                );
8680            }
8681        }
8682    }
8683}
8684
8685#[cfg(test)]
8686mod tests {
8687    #[tokio::test]
8688    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8689        let registry = Arc::new(Registry::default());
8690        let handler = ControlHandler::new(registry);
8691        assert_eq!(
8692            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8693            None
8694        );
8695        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8696        hello_via_sink(
8697            &handler,
8698            &ctx,
8699            &mut rx,
8700            hello_frame("daemon", PROTOCOL_VERSION, 1),
8701        )
8702        .await;
8703        assert_eq!(
8704            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8705            Some("daemon".into())
8706        );
8707        assert_eq!(
8708            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8709            None
8710        );
8711        assert_eq!(
8712            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8713            None,
8714            "a registered module is not necessarily supervised"
8715        );
8716        assert_eq!(
8717            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8718            None,
8719            "aggregate provenance has no single target"
8720        );
8721        assert_eq!(
8722            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8723            None
8724        );
8725    }
8726
8727    mod peer_manifest {
8728        include!("peer_manifest_tests.rs");
8729    }
8730    mod activity_read {
8731        include!("activity_read_tests.rs");
8732    }
8733
8734    mod lifecycle_activity {
8735        include!("control_lifecycle_tests.rs");
8736    }
8737    #[cfg(unix)]
8738    #[tokio::test]
8739    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8740        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8741        let path = dir.join("subc.jsonc");
8742        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8743            "program":"/bin/sleep","args":["60"],"protocol":"none",
8744            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8745        }}}).to_string()).unwrap();
8746        let mut configured = crate::daemon_config::load(&path)
8747            .unwrap()
8748            .unwrap()
8749            .modules
8750            .pop()
8751            .unwrap();
8752        let registry = std::sync::Arc::new(crate::Registry::default());
8753        let handle = crate::SupervisorHandle::new();
8754        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8755            .with_handle(handle.clone());
8756        let module = supervisor
8757            .supervise_configured_with_health(
8758                configured.module_spec(),
8759                true,
8760                configured.health.clone(),
8761                None,
8762                configured.restart,
8763            )
8764            .unwrap();
8765        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8766        let before = module.status().unwrap().pid;
8767        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8768        configured.health.cadence = std::time::Duration::from_secs(3600);
8769        let health_only = handler
8770            .reconcile_supervised_modules(
8771                &supervisor,
8772                vec![configured.clone()],
8773                false,
8774                Principal::Direct,
8775            )
8776            .await
8777            .unwrap();
8778        assert!(
8779            health_only.0.changed_pending_reload.is_empty(),
8780            "health policy is already applied live"
8781        );
8782        assert_eq!(module.status().unwrap().pid, before);
8783        assert_eq!(
8784            module.configuration().unwrap().1.http,
8785            configured.health.http
8786        );
8787        configured.args = vec!["61".into()];
8788        let launch = handler
8789            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8790            .await
8791            .unwrap();
8792        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8793        assert_eq!(
8794            module.status().unwrap().pid,
8795            before,
8796            "a launch edit is stored until reload"
8797        );
8798        module.drain().await.unwrap();
8799    }
8800    use cortexkit_test_support::ScratchDir;
8801    use std::{
8802        collections::BTreeMap,
8803        fmt,
8804        path::PathBuf,
8805        sync::{Arc, Mutex},
8806        time::Duration,
8807    };
8808
8809    use serde_json::{json, Value};
8810    use subc_protocol::{
8811        manifest::{
8812            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8813            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8814        },
8815        session::HealthStatus,
8816        FrameType,
8817    };
8818
8819    use super::*;
8820    use crate::{
8821        forwarding::{DataRoute, DataRouteState},
8822        registry::ChannelState,
8823        router::FrameSink,
8824        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8825        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8826        RouteCtx, Router,
8827    };
8828    use tokio::{
8829        sync::mpsc,
8830        time::{sleep, Instant},
8831    };
8832    use tracing::{
8833        field::{Field, Visit},
8834        Event, Subscriber,
8835    };
8836    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8837
8838    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8839    ///
8840    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8841    /// is only populated for `tests/*.rs` integration test binaries -- this file
8842    /// compiles as part of the library target, which gets neither. This test's
8843    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8844    /// and the sibling binary lives two directories up at
8845    /// `<target-dir>/<profile>/fake-aft-stub`.
8846    ///
8847    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8848    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8849    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8850    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8851    /// `NotFound`, which reads as a broken test rather than an unbuilt
8852    /// dependency -- so state the cause and the remedy instead. Deliberately a
8853    /// panic and not a silent skip: a test that quietly passes when it could not
8854    /// run is worse than one that fails, because it reports health it never
8855    /// verified.
8856    fn fake_aft_stub_path() -> PathBuf {
8857        let mut path = std::env::current_exe().expect("current_exe available in tests");
8858        path.pop(); // .../deps/
8859        path.pop(); // .../<profile>/
8860        path.push(if cfg!(windows) {
8861            "fake-aft-stub.exe"
8862        } else {
8863            "fake-aft-stub"
8864        });
8865        assert!(
8866            path.exists(),
8867            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8868             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8869            path.display()
8870        );
8871        path
8872    }
8873
8874    /// Whether clients retry `code` in place: the predicate itself, never a copy
8875    /// of its set. A copied list breaks silently when a code is added to or
8876    /// removed from the real one, and a stale copy here would let exactly the
8877    /// failure this test exists to catch pass.
8878    fn client_retries(code: &str) -> bool {
8879        subc_protocol::error_codes::is_retryable_route_open(code)
8880    }
8881
8882    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8883    /// of failure is worse than publishing none. A permanent fault dressed as
8884    /// retryable makes every client in the fleet retry forever against something
8885    /// that cannot recover; a transient fault dressed as permanent abandons work
8886    /// that would have succeeded.
8887    ///
8888    /// Asserting "a code exists" cannot catch either, because the string is free
8889    /// to say anything. This enumerates every variant and pins which side of the
8890    /// retry boundary it lands on, so a new variant must be classified here
8891    /// deliberately rather than inheriting whichever arm it was appended to.
8892    #[test]
8893    fn retryability_of_forwarding_codes_matches_the_failure() {
8894        // Transient by nature: the target is booting, reloading, or its endpoint
8895        // was swapped mid-flight. Retrying is how these resolve.
8896        let transient = [
8897            ForwardingError::NoModuleConnection,
8898            ForwardingError::ModuleReloading {
8899                module_id: "m".into(),
8900            },
8901            ForwardingError::StaleModuleEndpoint,
8902            ForwardingError::UnknownReservation {
8903                client_channel: 1,
8904                module_channel: 1,
8905            },
8906            ForwardingError::ConnectionClosing {
8907                connection_id: ConnectionId::new(1),
8908            },
8909            ForwardingError::ClientEgressClosed {
8910                connection_id: ConnectionId::new(1),
8911            },
8912            ForwardingError::ModuleEgressUnavailable {
8913                connection_id: ConnectionId::new(1),
8914            },
8915        ];
8916        for err in transient {
8917            let code = forwarding_error_code(&err);
8918            assert!(
8919                client_retries(code),
8920                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8921            );
8922        }
8923
8924        // Not fixed by retrying. Channel and correlation exhaustion need the
8925        // caller to close routes, and a poisoned lock is a daemon that cannot
8926        // recover at all — the worst thing to advertise as retryable, since every
8927        // client would storm a daemon that will never answer.
8928        let permanent = [
8929            ForwardingError::ConnectionRoleConflict {
8930                connection_id: ConnectionId::new(1),
8931            },
8932            ForwardingError::ClientRouteChannelExhausted {
8933                connection_id: ConnectionId::new(1),
8934            },
8935            ForwardingError::ModuleRouteChannelExhausted {
8936                endpoint: ModuleEndpointId {
8937                    connection_id: ConnectionId::new(1),
8938                    generation: 1,
8939                },
8940            },
8941            ForwardingError::RelayCorrelationExhausted,
8942            ForwardingError::RouteOpenBuild("x".into()),
8943            ForwardingError::Poisoned,
8944        ];
8945        for err in permanent {
8946            let code = forwarding_error_code(&err);
8947            assert!(
8948                !client_retries(code),
8949                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8950            );
8951        }
8952    }
8953
8954    /// The principal is the daemon's answer to "who is calling", and modules
8955    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8956    /// plexus gates connector invocation. So a stamp is an authorization input in
8957    /// another process, not a label — and both possible answers SUCCEED, which is
8958    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8959    /// first-party capability to something that never proved it; a supervised one
8960    /// stamped `Direct` silently strips a module of capability it is entitled to.
8961    ///
8962    /// Neither shows up in a test that only checks the bind succeeded. Before this
8963    /// test the only coverage was accidental —
8964    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8965    /// stamped principal on its way past, so narrowing that wire-shape test to its
8966    /// stated subject would have deleted the last assertion on this value. It
8967    /// still asserts the stamp, which is now redundancy rather than the only
8968    /// guard: both fail under the same mutation, and this one names the reason.
8969    /// SCOPE: this handler's supervisor has spawned nothing, so
8970    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8971    /// is unreachable here. Both assertions below are refusals, and a mutant that
8972    /// refuses everything would satisfy them.
8973    ///
8974    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8975    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8976    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8977    /// at source rather than assumed, since a citation is a claim about another
8978    /// file and ages like one. Recorded because a harness that structurally
8979    /// cannot reach an arm reports "none" for that arm identically to one that
8980    /// covers it and found nothing.
8981    #[tokio::test]
8982    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8983        let handler = ControlHandler::default();
8984        let frame =
8985            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8986
8987        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8988        // any process holding the connection file. Nothing was proved, so nothing
8989        // may be granted beyond the unattested floor.
8990        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8991        assert_eq!(
8992            stamped,
8993            Principal::Direct,
8994            "a caller that proved nothing must not be stamped as a supervised module"
8995        );
8996
8997        // A claimed module_id with a nonce no supervised child was given is a
8998        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8999        // quietly demoted to Direct, or an impersonation attempt looks identical
9000        // to an ordinary unattested connection.
9001        let forged = handler
9002            .route_open_principal(
9003                &frame,
9004                Some(ConsumerIdentity {
9005                    module_id: "aft".to_string(),
9006                    launch_nonce: "not-a-real-nonce".to_string(),
9007                }),
9008            )
9009            .unwrap();
9010        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9011        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9012    }
9013
9014    /// The test above hands `route_open_principal` an identity it built itself,
9015    /// which proves the stamping rule and nothing about where the identity comes
9016    /// from. The real producer is a wire body, and the two are joined by a serde
9017    /// field name that nothing else asserts.
9018    ///
9019    /// That join fails quietly in one specific way: an unrecognised key is simply
9020    /// absent after parsing, so a renamed or misspelled `consumer_identity`
9021    /// yields `None` and every supervised module silently drops to `Direct`.
9022    /// Capability-wise that is the safe direction, but it surfaces far from its
9023    /// cause — as a module mysteriously refused bash — and it would pass every
9024    /// test that builds its own input.
9025    ///
9026    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
9027    /// would break every client the moment the daemon gains a field, trading a
9028    /// quiet demotion for a hard refusal on additive change. Asserting the join
9029    /// instead means a rename breaks a test here rather than the fleet.
9030    #[test]
9031    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9032        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9033        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9034        let ClientControlRequest::RouteOpen {
9035            consumer_identity, ..
9036        } = parsed
9037        else {
9038            panic!("route.open body must parse as RouteOpen");
9039        };
9040        assert_eq!(
9041            consumer_identity,
9042            Some(ConsumerIdentity {
9043                module_id: "aft".to_string(),
9044                launch_nonce: "n".to_string(),
9045            }),
9046            "the wire field name must reach the value route_open_principal reads"
9047        );
9048    }
9049
9050    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9051        ModuleManifest::builder(module_id, "0.1.0")
9052            .protocol_ver(protocol_ver)
9053            .provides(vec![ProviderRole::ToolProvider {
9054                tools: vec![Tool {
9055                    name: "read".to_string(),
9056                    description: None,
9057                    execution_mode: ExecutionMode::Pure,
9058                    schema: json!({"type": "object"}),
9059                }],
9060                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9061                concurrency: Concurrency::ModuleManaged,
9062                emits_push: true,
9063                sub_supervises: true,
9064            }])
9065            .build()
9066    }
9067
9068    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9069        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9070    }
9071
9072    fn hello_frame_with_control_ops(
9073        module_id: &str,
9074        protocol_ver: u8,
9075        corr: u64,
9076        control_ops: Option<Vec<String>>,
9077    ) -> Frame {
9078        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9079    }
9080
9081    fn hello_frame_with_nonce(
9082        module_id: &str,
9083        protocol_ver: u8,
9084        corr: u64,
9085        launch_nonce: Option<&str>,
9086    ) -> Frame {
9087        hello_frame_full(
9088            module_id,
9089            protocol_ver,
9090            corr,
9091            None,
9092            launch_nonce.map(ToOwned::to_owned),
9093        )
9094    }
9095
9096    fn hello_frame_full(
9097        module_id: &str,
9098        protocol_ver: u8,
9099        corr: u64,
9100        control_ops: Option<Vec<String>>,
9101        launch_nonce: Option<String>,
9102    ) -> Frame {
9103        let body = serde_json::to_vec(&ModuleHelloBody {
9104            manifest: manifest(module_id, protocol_ver),
9105            protocol_ver,
9106            control_ops,
9107            launch_nonce,
9108        })
9109        .unwrap();
9110        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9111    }
9112
9113    fn hello_frame_with_rich_ui(
9114        module_id: &str,
9115        protocol_ver: u8,
9116        corr: u64,
9117        launch_nonce: Option<&str>,
9118        rich_ui: Value,
9119    ) -> Frame {
9120        let mut body = serde_json::to_value(ModuleHelloBody {
9121            manifest: manifest(module_id, protocol_ver),
9122            protocol_ver,
9123            control_ops: None,
9124            launch_nonce: launch_nonce.map(str::to_string),
9125        })
9126        .unwrap();
9127        body["manifest"]["rich_ui"] = rich_ui;
9128        Frame::build(
9129            FrameType::Hello,
9130            control_flags(),
9131            0,
9132            0,
9133            corr,
9134            serde_json::to_vec(&body).unwrap(),
9135        )
9136        .unwrap()
9137    }
9138
9139    fn non_routable_hello_frame_with_control_ops(
9140        module_id: &str,
9141        corr: u64,
9142        control_ops: Option<Vec<String>>,
9143    ) -> Frame {
9144        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9145        manifest.provides.clear();
9146        let body = serde_json::to_vec(&ModuleHelloBody {
9147            manifest,
9148            protocol_ver: PROTOCOL_VERSION,
9149            control_ops,
9150            launch_nonce: None,
9151        })
9152        .unwrap();
9153        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9154    }
9155
9156    fn capability_grammar_hello_frame(
9157        capabilities: Value,
9158        runtime_computed: Option<Value>,
9159        corr: u64,
9160    ) -> Frame {
9161        let mut body = serde_json::to_value(ModuleHelloBody {
9162            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9163            protocol_ver: PROTOCOL_VERSION,
9164            control_ops: None,
9165            launch_nonce: None,
9166        })
9167        .expect("HELLO body serializes");
9168        body["manifest"]["capabilities"] = capabilities;
9169        if let Some(runtime_computed) = runtime_computed {
9170            body["runtime_computed"] = runtime_computed;
9171        }
9172        Frame::build(
9173            FrameType::Hello,
9174            control_flags(),
9175            0,
9176            0,
9177            corr,
9178            serde_json::to_vec(&body).expect("HELLO body reserializes"),
9179        )
9180        .expect("HELLO frame builds")
9181    }
9182
9183    fn channel_request(channel: u16, corr: u64) -> Frame {
9184        Frame::build(
9185            FrameType::Request,
9186            Flags::new(true, Priority::Interactive, false),
9187            channel,
9188            0,
9189            corr,
9190            b"opaque".to_vec(),
9191        )
9192        .unwrap()
9193    }
9194
9195    fn route_ctx(
9196        connection_id: ConnectionId,
9197    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9198        let (tx, rx) = mpsc::channel(8);
9199        (
9200            RouteCtx {
9201                connection_id,
9202                egress: FrameSink::new(tx),
9203            },
9204            rx,
9205        )
9206    }
9207
9208    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9209        serde_json::from_slice(&frame.body).unwrap()
9210    }
9211
9212    /// Register a module over a connection that has a sink and return the
9213    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
9214    /// module's own sink rather than returning it as a reply, so the ack is
9215    /// taken off `rx` here and whatever the test reads next is what followed it.
9216    async fn hello_via_sink(
9217        handler: &ControlHandler,
9218        ctx: &RouteCtx,
9219        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9220        hello: Frame,
9221    ) -> Frame {
9222        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9223        assert!(
9224            replies.is_empty(),
9225            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9226        );
9227        let ack = rx
9228            .try_recv()
9229            .expect("HELLO_ACK is queued on the module sink")
9230            .frame;
9231        assert_eq!(ack.header.ty, FrameType::HelloAck);
9232        ack
9233    }
9234
9235    fn parse_error(frame: &Frame) -> Value {
9236        serde_json::from_slice(&frame.body).unwrap()
9237    }
9238
9239    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9240        serde_json::from_slice(&frame.body).unwrap()
9241    }
9242
9243    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9244        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9245            route_channel,
9246            route_epoch: 0,
9247            kind,
9248        })
9249        .unwrap();
9250        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9251    }
9252
9253    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9254        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9255            module_id: module_id.to_string(),
9256        })
9257        .unwrap();
9258        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9259    }
9260
9261    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9262        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9263    }
9264
9265    // Exercise retired opener fields without putting them back in BindIdentity.
9266    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9267        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9268        let mut frame = route_open_frame(corr, module_id, root);
9269        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9270        body["identity"]["project_root"] = serde_json::json!(path.as_path());
9271        frame.body = serde_json::to_vec(&body).unwrap();
9272        frame
9273    }
9274
9275    fn route_open_frame_with_consumer_capabilities(
9276        corr: u64,
9277        module_id: &str,
9278        _project_root: ScratchDir,
9279        consumer_capabilities: Option<Vec<String>>,
9280    ) -> Frame {
9281        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9282            busy_refusals: None,
9283            target: RouteTarget::ToolProvider {
9284                module_id: module_id.to_string(),
9285            },
9286            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9287            consumer_identity: None,
9288            consumer_capabilities,
9289            role_versions: None,
9290            admission_facts: None,
9291            scope: None,
9292        })
9293        .unwrap();
9294        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9295    }
9296
9297    fn route_open_frame_with_role_versions(
9298        corr: u64,
9299        module_id: &str,
9300        _project_root: ScratchDir,
9301        role_versions: Option<BTreeMap<String, String>>,
9302    ) -> Frame {
9303        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9304            busy_refusals: None,
9305            target: RouteTarget::ToolProvider {
9306                module_id: module_id.to_string(),
9307            },
9308            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9309            consumer_identity: None,
9310            consumer_capabilities: None,
9311            role_versions,
9312            admission_facts: None,
9313            scope: None,
9314        })
9315        .unwrap();
9316        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9317    }
9318
9319    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9320        entries
9321            .iter()
9322            .map(|(role, version)| (role.to_string(), version.to_string()))
9323            .collect()
9324    }
9325
9326    fn route_open_frame_with_admission_facts(
9327        corr: u64,
9328        module_id: &str,
9329        _project_root: ScratchDir,
9330        consumer_identity: Option<subc_control::ConsumerIdentity>,
9331        facts: Option<Value>,
9332    ) -> Frame {
9333        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9334            busy_refusals: None,
9335            target: RouteTarget::ToolProvider {
9336                module_id: module_id.to_string(),
9337            },
9338            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9339            consumer_identity,
9340            consumer_capabilities: None,
9341            role_versions: None,
9342            admission_facts: facts,
9343            scope: None,
9344        })
9345        .unwrap();
9346        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9347    }
9348
9349    #[derive(Clone, Default)]
9350    struct EventCapture {
9351        events: Arc<Mutex<Vec<CapturedEvent>>>,
9352    }
9353
9354    #[derive(Clone, Debug)]
9355    struct CapturedEvent {
9356        target: String,
9357        level: tracing::Level,
9358        fields: BTreeMap<String, String>,
9359    }
9360
9361    impl EventCapture {
9362        fn events(&self) -> Vec<CapturedEvent> {
9363            self.events.lock().unwrap().clone()
9364        }
9365    }
9366
9367    impl<S> Layer<S> for EventCapture
9368    where
9369        S: Subscriber,
9370    {
9371        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9372            let mut visitor = EventFieldVisitor::default();
9373            event.record(&mut visitor);
9374            self.events.lock().unwrap().push(CapturedEvent {
9375                target: event.metadata().target().to_string(),
9376                level: *event.metadata().level(),
9377                fields: visitor.fields,
9378            });
9379        }
9380    }
9381
9382    #[derive(Default)]
9383    struct EventFieldVisitor {
9384        fields: BTreeMap<String, String>,
9385    }
9386
9387    impl Visit for EventFieldVisitor {
9388        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9389            self.fields
9390                .insert(field.name().to_string(), format!("{value:?}"));
9391        }
9392    }
9393
9394    fn health_response(corr: u64, status: HealthStatus) -> Frame {
9395        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9396            status,
9397            detail: Some("warming".to_string()),
9398            metrics: Some(json!({"queue_depth": 3})),
9399        })
9400        .unwrap();
9401        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9402    }
9403
9404    fn route_bind_ack(corr: u64) -> Frame {
9405        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9406        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9407    }
9408
9409    fn unique_project_root(label: &str) -> ScratchDir {
9410        ScratchDir::new(label)
9411    }
9412
9413    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9414        match parse_route_poll(frame) {
9415            ClientControlResponse::RoutePoll {
9416                status: None,
9417                live: Some(live),
9418                ..
9419            } => assert_eq!(live, expected_live),
9420            other => panic!("unexpected route.poll response: {other:?}"),
9421        }
9422    }
9423
9424    fn bind_liveness_route(
9425        registry: &Registry,
9426        forwarding: &ForwardingTable,
9427        module_id: &str,
9428    ) -> (RouteCtx, u16, u32) {
9429        let module_connection = ConnectionId::new(101);
9430        let client_connection = ConnectionId::new(202);
9431        let registration = registry
9432            .register_with_control_ops(
9433                manifest(module_id, PROTOCOL_VERSION),
9434                PROTOCOL_VERSION,
9435                module_connection,
9436                module_baseline_control_ops(),
9437            )
9438            .unwrap();
9439        let (module_tx, _module_rx) = mpsc::channel(8);
9440        let endpoint = forwarding
9441            .register_module_connection(
9442                module_connection,
9443                module_id.to_string(),
9444                PROTOCOL_VERSION,
9445                manifest_concurrency(&registration.manifest),
9446                FrameSink::new(module_tx),
9447            )
9448            .unwrap();
9449        let (client_ctx, _client_rx) = route_ctx(client_connection);
9450        let pending = forwarding
9451            .begin_route_bind_relay_for_test(
9452                client_connection,
9453                client_ctx.egress.clone(),
9454                1,
9455                module_id,
9456            )
9457            .unwrap();
9458        assert_eq!(pending.endpoint, endpoint);
9459        let route_channel = pending.client_channel;
9460        let route_epoch = pending.client_epoch;
9461        forwarding
9462            .complete_pending_relay(
9463                module_connection,
9464                pending.corr,
9465                RouteBindRelayOutcome::Accepted,
9466            )
9467            .unwrap();
9468        (client_ctx, route_channel, route_epoch)
9469    }
9470
9471    struct FakeProcessLiveness {
9472        live: Option<bool>,
9473    }
9474
9475    impl ModuleProcessLiveness for FakeProcessLiveness {
9476        fn process_live(&self, _module_id: &str) -> Option<bool> {
9477            self.live
9478        }
9479    }
9480
9481    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9482    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9483    {
9484        let registry = Arc::new(Registry::default());
9485        let supervisor_handle = SupervisorHandle::new();
9486        let supervisor = Supervisor::new_for_test(
9487            Arc::clone(&registry),
9488            RestartPolicy::new(1, Duration::from_millis(10)),
9489        )
9490        .with_handle(supervisor_handle.clone());
9491        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9492        let module = supervisor
9493            .spawn(ModuleSpec {
9494                carry_containment: false,
9495                module_id: "stderr-tail-wire".to_string(),
9496                program: fake_aft_stub_path(),
9497                args: Vec::new(),
9498                env: vec![
9499                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9500                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9501                ],
9502                reserved: false,
9503                reserved_prefixes: Vec::new(),
9504                protocol: ModuleProtocol::Subc,
9505                overlap: Default::default(),
9506            })
9507            .unwrap();
9508
9509        let deadline = Instant::now() + Duration::from_secs(5);
9510        loop {
9511            let tail = module.stderr_tail(None, None);
9512            if tail
9513                .entries
9514                .iter()
9515                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9516                && tail.entries.iter().any(|entry| {
9517                    matches!(
9518                        entry,
9519                        TailEntry::Line {
9520                            truncated: true,
9521                            ..
9522                        }
9523                    )
9524                })
9525            {
9526                break;
9527            }
9528            assert!(
9529                Instant::now() < deadline,
9530                "module did not produce a truncated line and restart boundary: {tail:?}"
9531            );
9532            sleep(Duration::from_millis(10)).await;
9533        }
9534
9535        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9536        let request = ClientControlRequest::SupervisorStderrTail {
9537            module_id: "stderr-tail-wire".to_string(),
9538            max_lines: None,
9539            max_bytes: None,
9540        };
9541        let frame = Frame::build(
9542            FrameType::Request,
9543            control_flags(),
9544            0,
9545            0,
9546            1,
9547            serde_json::to_vec(&request).unwrap(),
9548        )
9549        .unwrap();
9550        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9551        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9552        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9553            serde_json::from_slice(&responses[0].body).unwrap()
9554        else {
9555            panic!("expected supervisor.stderr_tail response");
9556        };
9557
9558        assert!(
9559            tail.entries
9560                .iter()
9561                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9562            "the control response lost the restart boundary"
9563        );
9564        let Some(StderrTailEntry::Line {
9565            text,
9566            truncated,
9567            at_ms,
9568        }) = tail.entries.iter().find(|entry| {
9569            matches!(
9570                entry,
9571                StderrTailEntry::Line {
9572                    truncated: true,
9573                    ..
9574                }
9575            )
9576        })
9577        else {
9578            panic!("the control response lost the truncated line");
9579        };
9580        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9581        assert!(*truncated);
9582        assert!(
9583            at_ms.is_some(),
9584            "the control response lost the line's capture time"
9585        );
9586    }
9587
9588    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9589    /// read done on the worker thread would stall every other task until it
9590    /// finished; the read must run off the worker so this test's own task keeps
9591    /// running while the read is paused.
9592    #[tokio::test(flavor = "current_thread")]
9593    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9594        let dir = ScratchDir::new("terminals-off-worker");
9595        let journal_path = dir.join("terminals.jsonl");
9596        let registry = Arc::new(Registry::default());
9597        let supervisor_handle = SupervisorHandle::new();
9598        let supervisor =
9599            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9600                .with_handle(supervisor_handle.clone())
9601                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9602        let module = supervisor
9603            .spawn(ModuleSpec {
9604                carry_containment: false,
9605                module_id: "terminal-off-worker".to_string(),
9606                program: fake_aft_stub_path(),
9607                args: Vec::new(),
9608                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9609                reserved: false,
9610                reserved_prefixes: Vec::new(),
9611                protocol: ModuleProtocol::Subc,
9612                overlap: Default::default(),
9613            })
9614            .unwrap();
9615        let deadline = Instant::now() + Duration::from_secs(5);
9616        while module.terminal_history().entries.len() != 2 {
9617            assert!(Instant::now() < deadline, "module did not record two exits");
9618            sleep(Duration::from_millis(10)).await;
9619        }
9620
9621        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9622        let handler =
9623            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9624        let frame = Frame::build(
9625            FrameType::Request,
9626            control_flags(),
9627            0,
9628            0,
9629            1,
9630            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9631                module_id: "terminal-off-worker".to_string(),
9632            })
9633            .unwrap(),
9634        )
9635        .unwrap();
9636        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9637        let spawned_at = std::time::Instant::now();
9638        let read = tokio::spawn({
9639            let handler = Arc::clone(&handler);
9640            async move { handler.handle_control_frame(&ctx, frame).await }
9641        });
9642        // Waiting for the pause from a blocking thread keeps this task pending,
9643        // so the runtime's single worker is free to run the read task.
9644        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9645            .await
9646            .unwrap()
9647            .expect("the history read reached its pause");
9648        let elapsed = spawned_at.elapsed();
9649        assert!(
9650            elapsed < Duration::from_secs(2) && !read.is_finished(),
9651            "this task could not run while the history read was paused \
9652             (resumed after {elapsed:?}, read finished: {})",
9653            read.is_finished()
9654        );
9655
9656        drop(release);
9657        let responses = read.await.unwrap().unwrap();
9658        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9659        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9660            panic!("expected supervisor.terminals response");
9661        };
9662        assert_eq!(terminals.entries.len(), 2);
9663        assert_eq!(terminals.journal_skipped_lines, 0);
9664        assert_eq!(terminals.journal_read_errors, 0);
9665    }
9666
9667    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9668    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9669        let registry = Arc::new(Registry::default());
9670        let supervisor_handle = SupervisorHandle::new();
9671        let supervisor =
9672            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9673                .with_handle(supervisor_handle.clone());
9674        let module = supervisor
9675            .spawn(ModuleSpec {
9676                carry_containment: false,
9677                module_id: "terminal-golden".to_string(),
9678                program: fake_aft_stub_path(),
9679                args: Vec::new(),
9680                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9681                reserved: false,
9682                reserved_prefixes: Vec::new(),
9683                protocol: ModuleProtocol::Subc,
9684                overlap: Default::default(),
9685            })
9686            .unwrap();
9687
9688        let deadline = Instant::now() + Duration::from_secs(5);
9689        while module.terminal_history().entries.len() != 2 {
9690            assert!(
9691                Instant::now() < deadline,
9692                "module did not retain two terminal exits: {:?}",
9693                module.terminal_history()
9694            );
9695            sleep(Duration::from_millis(10)).await;
9696        }
9697
9698        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9699        let request = ClientControlRequest::SupervisorTerminals {
9700            module_id: "terminal-golden".to_string(),
9701        };
9702        let frame = Frame::build(
9703            FrameType::Request,
9704            control_flags(),
9705            0,
9706            0,
9707            1,
9708            serde_json::to_vec(&request).unwrap(),
9709        )
9710        .unwrap();
9711        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9712        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9713        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9714        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9715            panic!("expected supervisor.terminals response");
9716        };
9717        assert_eq!(terminals.entries.len(), 2);
9718        assert_eq!(terminals.dropped, 0);
9719
9720        let mut rendered = serde_json::to_value(response).unwrap();
9721        // Wall-clock fields are the observation contract, but not stable fixture
9722        // bytes; normalize only them after the real handler has shaped the response.
9723        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9724        for (index, entry) in rendered["entries"]
9725            .as_array_mut()
9726            .expect("terminal response entries array")
9727            .iter_mut()
9728            .enumerate()
9729        {
9730            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9731        }
9732
9733        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9734            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9735        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9736        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9737            std::fs::write(&golden_path, &serialized).unwrap();
9738        }
9739        let expected: Value =
9740            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9741        assert_eq!(rendered, expected);
9742    }
9743
9744    #[test]
9745    fn hello_registers_manifest_and_returns_ack() {
9746        let registry = Arc::new(Registry::default());
9747        let handler = ControlHandler::new(Arc::clone(&registry));
9748        let conn = ConnectionId::new(1);
9749
9750        let responses = handler
9751            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9752            .unwrap();
9753
9754        assert_eq!(responses.len(), 1);
9755        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9756        assert_eq!(responses[0].header.channel, 0);
9757        assert_eq!(responses[0].header.corr, 7);
9758        let ack = parse_ack(&responses[0]);
9759        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9760        assert!(ack
9761            .subc_capabilities
9762            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9763        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9764        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9765        assert!(ack
9766            .subc_ops
9767            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9768        assert!(ack
9769            .subc_ops
9770            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9771
9772        let registration = registry.get_module("aft").unwrap().unwrap();
9773        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9774        assert_eq!(registration.state, ChannelState::Active);
9775        assert_eq!(registration.connection_id, conn);
9776        assert_eq!(registration.control_ops, module_baseline_control_ops());
9777    }
9778
9779    #[test]
9780    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9781        let invalid_identifiers = [
9782            ("case_change", "credentials-Provider/v1"),
9783            ("leading_zero", "credentials-provider/v01"),
9784            ("trailing_hyphen", "credentials-provider-/v1"),
9785            ("consecutive_hyphens", "credentials--provider/v1"),
9786            ("uppercase", "Credentials-provider/v1"),
9787            ("missing_v", "credentials-provider/1"),
9788            ("whitespace", "credentials provider/v1"),
9789            ("zero_version", "credentials-provider/v0"),
9790            ("out_of_range_version", "credentials-provider/v4294967296"),
9791            (
9792                "overlength_name",
9793                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9794            ),
9795        ];
9796        let mut cases = invalid_identifiers
9797            .into_iter()
9798            .map(|(name, identifier)| {
9799                (
9800                    format!("identifier_{name}"),
9801                    "capabilities.provides[0]".to_string(),
9802                    identifier.to_string(),
9803                    json!({ "provides": [identifier] }),
9804                    None,
9805                )
9806            })
9807            .collect::<Vec<_>>();
9808        cases.extend([
9809            (
9810                "unknown_need".to_string(),
9811                "capabilities.requires[0].need".to_string(),
9812                "deferred".to_string(),
9813                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9814                None,
9815            ),
9816            (
9817                "duplicate_provides".to_string(),
9818                "capabilities.provides[1]".to_string(),
9819                "credentials-provider/v1".to_string(),
9820                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9821                None,
9822            ),
9823            (
9824                "duplicate_must_never_reach".to_string(),
9825                "capabilities.must_never_reach[1]".to_string(),
9826                "credentials-provider/v1".to_string(),
9827                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9828                None,
9829            ),
9830            (
9831                "duplicate_requires_same_need".to_string(),
9832                "capabilities.requires[1]".to_string(),
9833                "credentials-provider/v1".to_string(),
9834                json!({ "requires": [
9835                    { "capability": "credentials-provider/v1", "need": "required" },
9836                    { "capability": "credentials-provider/v1", "need": "required" }
9837                ] }),
9838                None,
9839            ),
9840            (
9841                "duplicate_requires_conflicting_need".to_string(),
9842                "capabilities.requires[1]".to_string(),
9843                "credentials-provider/v1".to_string(),
9844                json!({ "requires": [
9845                    { "capability": "credentials-provider/v1", "need": "required" },
9846                    { "capability": "credentials-provider/v1", "need": "optional" }
9847                ] }),
9848                None,
9849            ),
9850            (
9851                "capabilities_root_pointer".to_string(),
9852                "runtime_computed[0]".to_string(),
9853                "/capabilities".to_string(),
9854                json!({}),
9855                Some(json!(["/capabilities"])),
9856            ),
9857            (
9858                "capabilities_descendant_pointer".to_string(),
9859                "runtime_computed[0]".to_string(),
9860                "/capabilities/provides".to_string(),
9861                json!({}),
9862                Some(json!(["/capabilities/provides"])),
9863            ),
9864            (
9865                "malformed_pointer_without_leading_slash".to_string(),
9866                "runtime_computed[0]".to_string(),
9867                "capabilities".to_string(),
9868                json!({}),
9869                Some(json!(["capabilities"])),
9870            ),
9871            (
9872                "malformed_pointer_escape".to_string(),
9873                "runtime_computed[0]".to_string(),
9874                "/roles/~2/tools".to_string(),
9875                json!({}),
9876                Some(json!(["/roles/~2/tools"])),
9877            ),
9878            (
9879                "unknown_capabilities_field".to_string(),
9880                "capabilities.future".to_string(),
9881                "<array>".to_string(),
9882                json!({ "future": [] }),
9883                None,
9884            ),
9885        ]);
9886
9887        for (index, (name, field, value, capabilities, runtime_computed)) in
9888            cases.into_iter().enumerate()
9889        {
9890            let registry = Arc::new(Registry::default());
9891            let handler = ControlHandler::new(Arc::clone(&registry));
9892            let response = handler
9893                .handle_control(
9894                    ConnectionId::new((index + 1) as u64),
9895                    capability_grammar_hello_frame(
9896                        capabilities,
9897                        runtime_computed,
9898                        index as u64 + 1,
9899                    ),
9900                )
9901                .expect("invalid HELLO returns a refusal");
9902
9903            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9904            let error = parse_error(&response[0]);
9905            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9906            let message = error["message"]
9907                .as_str()
9908                .expect("error message is a string");
9909            assert!(
9910                message.contains(&field),
9911                "{name}: field missing from {message}"
9912            );
9913            assert!(
9914                message.contains(&value),
9915                "{name}: value missing from {message}"
9916            );
9917            assert_eq!(
9918                registry
9919                    .active_registration_count()
9920                    .expect("registry reads"),
9921                0,
9922                "{name}: refused HELLO must not create a catalog entry"
9923            );
9924        }
9925    }
9926
9927    #[test]
9928    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9929        let registry = Arc::new(Registry::default());
9930        let handler = ControlHandler::new(Arc::clone(&registry));
9931        let capabilities = json!({
9932            "provides": ["credentials-provider/v1"],
9933            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9934            "must_never_reach": ["federation-transport/v1"]
9935        });
9936        let response = handler
9937            .handle_control(
9938                ConnectionId::new(99),
9939                capability_grammar_hello_frame(
9940                    capabilities.clone(),
9941                    Some(json!(["/roles/0/tools"])),
9942                    99,
9943                ),
9944            )
9945            .expect("valid HELLO registers");
9946        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9947
9948        let request = Frame::build(
9949            FrameType::Request,
9950            control_flags(),
9951            0,
9952            0,
9953            100,
9954            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9955                .expect("catalog request serializes"),
9956        )
9957        .expect("catalog request frame builds");
9958        let response = handler
9959            .handle_catalog_list(request, None)
9960            .expect("catalog list succeeds");
9961        let ClientControlResponse::CatalogList { modules, .. } =
9962            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9963        else {
9964            panic!("catalog request must return catalog.list");
9965        };
9966        assert_eq!(modules.len(), 1);
9967        assert_eq!(
9968            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9969            capabilities
9970        );
9971    }
9972
9973    #[test]
9974    fn catalog_list_mirrors_management_operation_description() {
9975        let registry = Arc::new(Registry::default());
9976        let handler = ControlHandler::new(Arc::clone(&registry));
9977        let description = "List managed records and return their identifiers and metadata.";
9978        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9979        manifest.provides = vec![ProviderRole::ManagementSurface {
9980            operations: vec![ManagementOperation {
9981                name: "records.list".to_string(),
9982                kind: ManagementOperationKind::Query,
9983                description: Some(description.to_string()),
9984            }],
9985            config_schema: json!({"type": "object"}),
9986            observability: vec![ObservabilitySurface {
9987                name: "records.stats".to_string(),
9988                kind: ObservabilityKind::Snapshot,
9989            }],
9990            identity_scope: vec![IdentityScope::Project],
9991            concurrency: Concurrency::ModuleManaged,
9992        }];
9993        registry
9994            .register_with_control_ops(
9995                manifest,
9996                PROTOCOL_VERSION,
9997                ConnectionId::new(99),
9998                Vec::new(),
9999            )
10000            .expect("described management manifest registers");
10001
10002        let request = Frame::build(
10003            FrameType::Request,
10004            control_flags(),
10005            0,
10006            0,
10007            100,
10008            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10009                .expect("catalog request serializes"),
10010        )
10011        .expect("catalog request frame builds");
10012        let response = handler
10013            .handle_catalog_list(request, None)
10014            .expect("catalog list succeeds");
10015        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10016        assert_eq!(
10017            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10018            "catalog.list must preserve the declared operation description verbatim"
10019        );
10020    }
10021
10022    #[test]
10023    fn catalog_list_mirrors_registered_operation_delivery() {
10024        let registry = Arc::new(Registry::default());
10025        let handler = ControlHandler::new(Arc::clone(&registry));
10026        let delivery =
10027            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10028        let mut body = serde_json::to_value(ModuleHelloBody {
10029            manifest: manifest("watch-management", PROTOCOL_VERSION),
10030            protocol_ver: PROTOCOL_VERSION,
10031            control_ops: None,
10032            launch_nonce: None,
10033        })
10034        .unwrap();
10035        body["manifest"]["provides"] = json!([{
10036            "role": "management_surface",
10037            "operations": [
10038                {"name": "status", "kind": "query"},
10039                {"name": "future", "kind": "query"},
10040                {"name": "reset", "kind": "mutate"}
10041            ],
10042            "config_schema": {}, "observability": [], "identity_scope": []
10043        }]);
10044        body["manifest"]["operation_delivery"] = delivery.clone();
10045        let hello = Frame::build(
10046            FrameType::Hello,
10047            control_flags(),
10048            0,
10049            0,
10050            99,
10051            serde_json::to_vec(&body).unwrap(),
10052        )
10053        .unwrap();
10054        let response = handler
10055            .handle_control(ConnectionId::new(99), hello)
10056            .unwrap();
10057        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10058
10059        let (_, registrations) = registry.list_modules().unwrap();
10060        let registered = &registrations[0].manifest;
10061        assert!(registered.is_watch_operation("status"));
10062        assert!(!registered.is_watch_operation("future"));
10063        assert!(!registered.is_watch_operation("reset"));
10064        assert!(!registered.is_watch_operation("missing"));
10065
10066        let request = Frame::build(
10067            FrameType::Request,
10068            control_flags(),
10069            0,
10070            0,
10071            100,
10072            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10073        )
10074        .unwrap();
10075        let response = handler.handle_catalog_list(request, None).unwrap();
10076        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10077        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10078        let ClientControlResponse::CatalogList { modules, .. } =
10079            serde_json::from_slice(&response[0].body).unwrap()
10080        else {
10081            panic!("expected catalog.list");
10082        };
10083        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10084    }
10085
10086    #[test]
10087    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10088        let registry = Arc::new(Registry::default());
10089        let supervisor = SupervisorHandle::new();
10090        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10091        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10092        let descriptor = json!({
10093            "schema": "ckios.richui.manifest/1",
10094            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10095            "bytes": 4096,
10096            "chunks": 4
10097        });
10098        let connection = ConnectionId::new(201);
10099        let hello = handler
10100            .handle_control(
10101                connection,
10102                hello_frame_with_rich_ui(
10103                    "vault",
10104                    PROTOCOL_VERSION,
10105                    201,
10106                    Some("the-real-nonce"),
10107                    descriptor.clone(),
10108                ),
10109            )
10110            .unwrap();
10111        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10112
10113        let request =
10114            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10115        let provides = registry
10116            .get_module_by_connection(connection)
10117            .unwrap()
10118            .unwrap()
10119            .manifest
10120            .provides;
10121        handler
10122            .handle_catalog_update(connection, request, provides, None, Some(false))
10123            .unwrap();
10124
10125        let request = Frame::build(
10126            FrameType::Request,
10127            control_flags(),
10128            0,
10129            0,
10130            203,
10131            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10132        )
10133        .unwrap();
10134        let response = handler.handle_catalog_list(request, None).unwrap();
10135        let ClientControlResponse::CatalogList { modules, .. } =
10136            serde_json::from_slice(&response[0].body).unwrap()
10137        else {
10138            panic!("expected catalog.list");
10139        };
10140        assert_eq!(
10141            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10142            descriptor
10143        );
10144        assert!(modules[0].nonce_verified);
10145    }
10146
10147    fn hello_with_role_declarations(declarations: Value) -> Frame {
10148        let mut body = serde_json::to_value(ModuleHelloBody {
10149            manifest: manifest("role-test", PROTOCOL_VERSION),
10150            protocol_ver: PROTOCOL_VERSION,
10151            control_ops: None,
10152            launch_nonce: None,
10153        })
10154        .unwrap();
10155        body["manifest"]["role_declarations"] = declarations;
10156        Frame::build(
10157            FrameType::Hello,
10158            control_flags(),
10159            0,
10160            0,
10161            301,
10162            serde_json::to_vec(&body).unwrap(),
10163        )
10164        .unwrap()
10165    }
10166
10167    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10168        let request = Frame::build(
10169            FrameType::Request,
10170            control_flags(),
10171            0,
10172            0,
10173            302,
10174            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10175        )
10176        .unwrap();
10177        let response = handler.handle_catalog_list(request, None).unwrap();
10178        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10179        let ClientControlResponse::CatalogList { mut modules, .. } =
10180            serde_json::from_slice(&response[0].body).unwrap()
10181        else {
10182            panic!("expected catalog.list");
10183        };
10184        (raw["modules"][0].clone(), modules.remove(0))
10185    }
10186
10187    #[test]
10188    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10189        let registry = Arc::new(Registry::default());
10190        let handler = ControlHandler::new(Arc::clone(&registry));
10191        let declarations = json!({
10192            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10193            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10194        });
10195        let connection = ConnectionId::new(301);
10196        let response = handler
10197            .handle_control(
10198                connection,
10199                hello_with_role_declarations(declarations.clone()),
10200            )
10201            .unwrap();
10202        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10203        let registered = registry
10204            .get_module_by_connection(connection)
10205            .unwrap()
10206            .unwrap()
10207            .manifest;
10208        // Compared as JSON values, not bytes: object key order is not part of
10209        // the relay's promise, and workspace builds unify serde_json's
10210        // preserve_order feature, which changes the order a literal keeps.
10211        assert_eq!(
10212            serde_json::to_value(&registered.role_declarations).unwrap(),
10213            declarations
10214        );
10215        let (raw, entry) = role_declarations_catalog(&handler);
10216        assert_eq!(raw["role_declarations"], declarations);
10217        assert_eq!(entry.role_declarations, registered.role_declarations);
10218
10219        let request =
10220            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10221        handler
10222            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10223            .unwrap();
10224        let (raw, entry) = role_declarations_catalog(&handler);
10225        assert_eq!(raw["role_declarations"], declarations);
10226        assert_eq!(entry.role_declarations, registered.role_declarations);
10227    }
10228
10229    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10230        let registry = Arc::new(Registry::default());
10231        let handler = ControlHandler::new(Arc::clone(&registry));
10232        let capture = EventCapture::default();
10233        let _subscriber =
10234            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10235        let response = handler
10236            .handle_control(
10237                ConnectionId::new(301),
10238                hello_with_role_declarations(declarations),
10239            )
10240            .unwrap();
10241        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10242        assert!(registry
10243            .get_module_by_connection(ConnectionId::new(301))
10244            .unwrap()
10245            .unwrap()
10246            .manifest
10247            .role_declarations
10248            .is_empty());
10249        let (raw, entry) = role_declarations_catalog(&handler);
10250        assert!(raw.get("role_declarations").is_none());
10251        assert!(entry.role_declarations.is_empty());
10252        let warnings: Vec<_> = capture
10253            .events()
10254            .into_iter()
10255            .filter(|event| {
10256                event.fields.get("message").map(String::as_str)
10257                    == Some("dropping whole role_declarations map from HELLO")
10258            })
10259            .collect();
10260        assert_eq!(warnings.len(), 1);
10261        assert_eq!(warnings[0].level, tracing::Level::WARN);
10262        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10263        assert_eq!(warnings[0].fields["reason"], reason);
10264    }
10265
10266    #[test]
10267    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10268        for value in [
10269            json!(null),
10270            json!(false),
10271            json!(0),
10272            json!("opaque"),
10273            json!([]),
10274        ] {
10275            assert_hello_drops_role_declarations_with_warning(
10276                json!({"routing/v1": {}, "broken/v1": value}),
10277                "role_declarations values must be objects",
10278            );
10279        }
10280        assert_hello_drops_role_declarations_with_warning(
10281            json!([]),
10282            "role_declarations must be an object",
10283        );
10284    }
10285
10286    #[test]
10287    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10288        assert_hello_drops_role_declarations_with_warning(
10289            json!({"routing/v1": {}, "bad key": {}}),
10290            "role_declarations key must use capability-name grammar",
10291        );
10292    }
10293
10294    #[test]
10295    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10296        assert_hello_drops_role_declarations_with_warning(
10297            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10298            "role_declarations exceeds 16384 compact JSON bytes",
10299        );
10300    }
10301
10302    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10303        let mut body = serde_json::to_value(ModuleHelloBody {
10304            manifest: manifest("flow-ops", PROTOCOL_VERSION),
10305            protocol_ver: PROTOCOL_VERSION,
10306            control_ops: None,
10307            launch_nonce: None,
10308        })
10309        .unwrap();
10310        body["manifest"]["provides"] = json!([{
10311            "role": "management_surface",
10312            "operations": [
10313                {"name": "diagnostics.findings", "kind": "query"},
10314                {"name": "reset", "kind": "mutate"}
10315            ],
10316            "config_schema": {}, "observability": [], "identity_scope": []
10317        }]);
10318        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10319        if let Some(flow_operations) = flow_operations {
10320            body["manifest"]["flow_operations"] = flow_operations;
10321        }
10322        Frame::build(
10323            FrameType::Hello,
10324            control_flags(),
10325            0,
10326            0,
10327            401,
10328            serde_json::to_vec(&body).unwrap(),
10329        )
10330        .unwrap()
10331    }
10332
10333    #[test]
10334    fn catalog_list_mirrors_flow_operations_unchanged() {
10335        // Order is kept as declared (not sorted) and an empty list stays an
10336        // affirmative "none", distinct from an absent declaration.
10337        for (capability, flow_operations) in [
10338            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10339            ("call-modes/v1", json!(["diagnostics.findings"])),
10340            ("flow-scopes/v1", json!([])),
10341        ] {
10342            let registry = Arc::new(Registry::default());
10343            let handler = ControlHandler::new(Arc::clone(&registry));
10344            let connection = ConnectionId::new(401);
10345            let response = handler
10346                .handle_control(
10347                    connection,
10348                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10349                )
10350                .unwrap();
10351            assert_eq!(response[0].header.ty, FrameType::HelloAck);
10352            let registered = registry
10353                .get_module_by_connection(connection)
10354                .unwrap()
10355                .unwrap()
10356                .manifest;
10357            assert_eq!(
10358                serde_json::to_value(&registered.flow_operations).unwrap(),
10359                flow_operations
10360            );
10361            let (raw, entry) = role_declarations_catalog(&handler);
10362            assert_eq!(raw["flow_operations"], flow_operations);
10363            assert_eq!(entry.flow_operations, registered.flow_operations);
10364        }
10365
10366        let registry = Arc::new(Registry::default());
10367        let handler = ControlHandler::new(Arc::clone(&registry));
10368        let response = handler
10369            .handle_control(
10370                ConnectionId::new(401),
10371                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10372            )
10373            .unwrap();
10374        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10375        let (raw, entry) = role_declarations_catalog(&handler);
10376        assert!(raw.get("flow_operations").is_none());
10377        assert_eq!(entry.flow_operations, None);
10378    }
10379
10380    fn assert_hello_drops_flow_operations_with_warning(
10381        capabilities: Value,
10382        flow_operations: Value,
10383        reason: &str,
10384    ) {
10385        let registry = Arc::new(Registry::default());
10386        let handler = ControlHandler::new(Arc::clone(&registry));
10387        let capture = EventCapture::default();
10388        let _subscriber =
10389            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10390        let response = handler
10391            .handle_control(
10392                ConnectionId::new(401),
10393                hello_with_flow_operations(capabilities, Some(flow_operations)),
10394            )
10395            .unwrap();
10396        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10397        let registered = registry
10398            .get_module_by_connection(ConnectionId::new(401))
10399            .unwrap()
10400            .expect("the module still registers")
10401            .manifest;
10402        assert_eq!(registered.module_id, "flow-ops");
10403        assert_eq!(registered.flow_operations, None);
10404        let (raw, entry) = role_declarations_catalog(&handler);
10405        assert_eq!(raw["module_id"], "flow-ops");
10406        assert!(raw.get("flow_operations").is_none());
10407        assert_eq!(entry.flow_operations, None);
10408        let warnings: Vec<_> = capture
10409            .events()
10410            .into_iter()
10411            .filter(|event| {
10412                event.fields.get("message").map(String::as_str)
10413                    == Some("dropping flow_operations from HELLO")
10414            })
10415            .collect();
10416        assert_eq!(warnings.len(), 1);
10417        assert_eq!(warnings[0].level, tracing::Level::WARN);
10418        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10419        assert_eq!(warnings[0].fields["reason"], reason);
10420    }
10421
10422    #[test]
10423    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10424        assert_hello_drops_flow_operations_with_warning(
10425            json!(["flow-scopes/v1"]),
10426            json!(["diagnostics.findings", "missing"]),
10427            "flow_operations names 'missing', which is not a declared management operation",
10428        );
10429    }
10430
10431    #[test]
10432    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10433        assert_hello_drops_flow_operations_with_warning(
10434            json!([]),
10435            json!(["diagnostics.findings"]),
10436            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10437        );
10438        // A run-only capability does not admit flows either.
10439        assert_hello_drops_flow_operations_with_warning(
10440            json!(["agent-run-scopes/v1"]),
10441            json!([]),
10442            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10443        );
10444    }
10445
10446    #[test]
10447    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10448        assert_hello_drops_flow_operations_with_warning(
10449            json!(["flow-scopes/v1"]),
10450            json!(["reset", "reset"]),
10451            "flow_operations names 'reset' more than once",
10452        );
10453        assert_hello_drops_flow_operations_with_warning(
10454            json!(["flow-scopes/v1"]),
10455            json!(vec!["reset"; 257]),
10456            "flow_operations lists 257 names, more than 256",
10457        );
10458        assert_hello_drops_flow_operations_with_warning(
10459            json!(["flow-scopes/v1"]),
10460            json!("reset"),
10461            "flow_operations must be an array of operation names",
10462        );
10463    }
10464
10465    #[test]
10466    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10467        let registry = Arc::new(Registry::default());
10468        let handler = ControlHandler::new(registry.clone());
10469        let descriptor = json!({
10470            "schema": "ckios.richui.manifest/1",
10471            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10472            "bytes": 128,
10473            "chunks": 2
10474        });
10475        let malformed = [
10476            ("schema", json!("")),
10477            ("schema", json!("a".repeat(65))),
10478            ("schema", json!("Bad schema")),
10479            (
10480                "manifest_digest",
10481                json!(format!("sha256:{}", "A".repeat(64))),
10482            ),
10483            ("manifest_digest", json!("sha256:abc")),
10484            ("bytes", json!(0)),
10485            ("bytes", json!(16_777_217)),
10486            ("bytes", json!("128")),
10487            ("chunks", json!(0)),
10488            ("chunks", json!(1025)),
10489            ("chunks", json!("2")),
10490        ];
10491        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10492            let mut invalid = descriptor.clone();
10493            invalid[field] = invalid_value;
10494            let response = handler
10495                .handle_control(
10496                    ConnectionId::new(210 + index as u64),
10497                    hello_frame_with_rich_ui(
10498                        &format!("bad-rich-ui-{index}"),
10499                        PROTOCOL_VERSION,
10500                        210 + index as u64,
10501                        None,
10502                        invalid,
10503                    ),
10504                )
10505                .unwrap();
10506            assert_eq!(response[0].header.ty, FrameType::Error);
10507            let error = parse_error(&response[0]);
10508            assert_eq!(error["code"], "invalid_manifest");
10509            assert!(error["message"]
10510                .as_str()
10511                .unwrap()
10512                .contains(&format!("rich_ui.{field}")));
10513            assert!(registry
10514                .get_module(&format!("bad-rich-ui-{index}"))
10515                .unwrap()
10516                .is_none());
10517        }
10518    }
10519
10520    #[test]
10521    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10522        let registry = Arc::new(Registry::default());
10523        let handler = ControlHandler::new(Arc::clone(&registry));
10524        let connection = ConnectionId::new(219);
10525        registry
10526            .register_with_control_ops(
10527                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10528                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10529                        "bad schema",
10530                        format!("sha256:{}", "a".repeat(64)),
10531                        128,
10532                        2,
10533                    )))
10534                    .build(),
10535                PROTOCOL_VERSION,
10536                connection,
10537                Vec::new(),
10538            )
10539            .unwrap();
10540        let request =
10541            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10542        let response = handler
10543            .handle_catalog_update(connection, request, Vec::new(), None, None)
10544            .unwrap();
10545        assert_eq!(response[0].header.ty, FrameType::Error);
10546        let error = parse_error(&response[0]);
10547        assert_eq!(error["code"], "invalid_manifest");
10548        assert!(error["message"]
10549            .as_str()
10550            .unwrap()
10551            .contains("rich_ui.schema"));
10552    }
10553
10554    #[test]
10555    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10556        let registry = Arc::new(Registry::default());
10557        let supervisor = SupervisorHandle::new();
10558        supervisor.open_swap("provider", "candidate-nonce".to_string());
10559        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10560        let connection = ConnectionId::new(218);
10561        let hello = handler
10562            .handle_control(
10563                connection,
10564                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10565            )
10566            .unwrap();
10567        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10568        assert!(registry.list_modules().unwrap().1.is_empty());
10569
10570        registry.promote_candidate("provider").unwrap().unwrap();
10571        let request = Frame::build(
10572            FrameType::Request,
10573            control_flags(),
10574            0,
10575            0,
10576            219,
10577            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10578        )
10579        .unwrap();
10580        let response = handler.handle_catalog_list(request, None).unwrap();
10581        let ClientControlResponse::CatalogList { modules, .. } =
10582            serde_json::from_slice(&response[0].body).unwrap()
10583        else {
10584            panic!("expected catalog.list");
10585        };
10586        assert_eq!(modules.len(), 1);
10587        assert!(modules[0].nonce_verified);
10588    }
10589
10590    #[test]
10591    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10592        for provenance_json in [
10593            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10594            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10595        ] {
10596            let registry = Arc::new(Registry::default());
10597            let handler = ControlHandler::new(registry);
10598            let hello = Frame::build(
10599                FrameType::Hello,
10600                control_flags(),
10601                0,
10602                0,
10603                220,
10604                serde_json::to_vec(&ModuleHelloBody {
10605                    manifest: ModuleManifest::builder("built", "0.1.0")
10606                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10607                        .build(),
10608                    protocol_ver: PROTOCOL_VERSION,
10609                    control_ops: None,
10610                    launch_nonce: None,
10611                })
10612                .unwrap(),
10613            )
10614            .unwrap();
10615            assert_eq!(
10616                handler
10617                    .handle_control(ConnectionId::new(220), hello)
10618                    .unwrap()[0]
10619                    .header
10620                    .ty,
10621                FrameType::HelloAck
10622            );
10623            let request = Frame::build(
10624                FrameType::Request,
10625                control_flags(),
10626                0,
10627                0,
10628                221,
10629                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10630            )
10631            .unwrap();
10632            let response = handler.handle_catalog_list(request, None).unwrap();
10633            let ClientControlResponse::CatalogList { modules, .. } =
10634                serde_json::from_slice(&response[0].body).unwrap()
10635            else {
10636                panic!("expected catalog.list");
10637            };
10638            assert_eq!(modules.len(), 1);
10639            assert_eq!(
10640                serde_json::to_vec(&modules[0]).unwrap(),
10641                format!(
10642                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10643                )
10644                .as_bytes()
10645            );
10646        }
10647    }
10648
10649    #[test]
10650    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10651        let registry = Arc::new(Registry::default());
10652        let handler = ControlHandler::new(Arc::clone(&registry));
10653        let hello = Frame::build(
10654            FrameType::Hello,
10655            control_flags(),
10656            0,
10657            0,
10658            220,
10659            serde_json::to_vec(&ModuleHelloBody {
10660                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10661                protocol_ver: PROTOCOL_VERSION,
10662                control_ops: None,
10663                launch_nonce: Some("unverified".to_string()),
10664            })
10665            .unwrap(),
10666        )
10667        .unwrap();
10668        assert_eq!(
10669            handler
10670                .handle_control(ConnectionId::new(220), hello)
10671                .unwrap()[0]
10672                .header
10673                .ty,
10674            FrameType::HelloAck
10675        );
10676        let request = Frame::build(
10677            FrameType::Request,
10678            control_flags(),
10679            0,
10680            0,
10681            221,
10682            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10683        )
10684        .unwrap();
10685        let response = handler.handle_catalog_list(request, None).unwrap();
10686        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10687        assert!(wire["modules"][0].get("provenance").is_none());
10688        let ClientControlResponse::CatalogList { modules, .. } =
10689            serde_json::from_slice(&response[0].body).unwrap()
10690        else {
10691            panic!("expected catalog.list");
10692        };
10693        assert_eq!(
10694            serde_json::to_vec(&modules[0]).unwrap(),
10695            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10696        );
10697        assert!(!modules[0].nonce_verified);
10698        assert!(modules[0].provenance.is_none());
10699    }
10700
10701    #[test]
10702    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10703        let registry = Arc::new(Registry::default());
10704        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10705            [("vault".to_string(), true), ("squatter".to_string(), true)],
10706            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10707        );
10708        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10709        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10710            provides: vec!["credentials-provider/v1".to_string()],
10711            requires: Vec::new(),
10712            must_never_reach: Vec::new(),
10713        });
10714        let frame = Frame::build(
10715            FrameType::Hello,
10716            control_flags(),
10717            0,
10718            0,
10719            77,
10720            serde_json::to_vec(&ModuleHelloBody {
10721                manifest: squatter,
10722                protocol_ver: PROTOCOL_VERSION,
10723                control_ops: None,
10724                launch_nonce: None,
10725            })
10726            .expect("HELLO serializes"),
10727        )
10728        .expect("HELLO frame builds");
10729        let response = handler
10730            .handle_control(ConnectionId::new(77), frame)
10731            .expect("reserved claim receives a typed refusal");
10732        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10733        assert_eq!(
10734            registry
10735                .active_registration_count()
10736                .expect("registry reads"),
10737            0,
10738            "a reserved capability refusal must not leave a catalog entry"
10739        );
10740    }
10741
10742    #[test]
10743    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10744        for settlement in ["timeout", "inconclusive", "drop"] {
10745            let breakers = RouteBindBreakers::default();
10746            let RouteBindAdmission::Admitted {
10747                guard: mut old,
10748                probe: false,
10749            } = breakers.admit("prov")
10750            else {
10751                panic!("ordinary relay admitted")
10752            };
10753            let RouteBindAdmission::Admitted {
10754                guard: mut opener, ..
10755            } = breakers.admit("prov")
10756            else {
10757                panic!("second relay admitted")
10758            };
10759            assert!(
10760                !opener
10761                    .record_timeout(1, Duration::ZERO)
10762                    .unwrap()
10763                    .reopened_after_probe
10764            );
10765            let RouteBindAdmission::Admitted {
10766                guard: mut probe,
10767                probe: true,
10768            } = breakers.admit("prov")
10769            else {
10770                panic!("one cooldown probe admitted")
10771            };
10772            match settlement {
10773                "timeout" => assert!(
10774                    !old.record_timeout(1, Duration::ZERO)
10775                        .unwrap()
10776                        .reopened_after_probe
10777                ),
10778                "inconclusive" => old.record_inconclusive(),
10779                "drop" => drop(old),
10780                _ => unreachable!(),
10781            }
10782            assert!(
10783                matches!(
10784                    breakers.admit("prov"),
10785                    RouteBindAdmission::Refused {
10786                        probe_in_flight: true,
10787                        ..
10788                    }
10789                ),
10790                "{settlement} of a pre-open relay cannot release the real probe"
10791            );
10792            assert!(
10793                probe
10794                    .record_timeout(1, Duration::ZERO)
10795                    .unwrap()
10796                    .reopened_after_probe
10797            );
10798            assert!(matches!(
10799                breakers.admit("prov"),
10800                RouteBindAdmission::Admitted { probe: true, .. }
10801            ));
10802        }
10803        let breakers = RouteBindBreakers::default();
10804        let admit = || match breakers.admit("prov") {
10805            RouteBindAdmission::Admitted { guard, .. } => guard,
10806            _ => panic!("relay admitted"),
10807        };
10808        admit().record_timeout(1, Duration::ZERO);
10809        let mut old_probe = admit();
10810        breakers.reset_for_new_module_connection("prov");
10811        admit().record_timeout(1, Duration::ZERO);
10812        let _new_probe = admit();
10813        old_probe.record_inconclusive();
10814        assert!(matches!(
10815            breakers.admit("prov"),
10816            RouteBindAdmission::Refused {
10817                probe_in_flight: true,
10818                ..
10819            }
10820        ));
10821    }
10822
10823    #[tokio::test]
10824    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10825        for candidate in [false, true] {
10826            let registry = Arc::new(Registry::default());
10827            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10828                [("vault".to_string(), true), ("squatter".to_string(), true)],
10829                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10830            );
10831            let conn = ConnectionId::new(77);
10832            let (ctx, mut rx) = route_ctx(conn);
10833            let initial = capability_manifest("squatter", &[], &[]);
10834            if candidate {
10835                registry
10836                    .register_candidate_with_control_ops(
10837                        initial.clone(),
10838                        PROTOCOL_VERSION,
10839                        conn,
10840                        module_baseline_control_ops(),
10841                    )
10842                    .unwrap();
10843                handler
10844                    .forwarding
10845                    .register_candidate_module_connection(
10846                        conn,
10847                        "squatter".to_string(),
10848                        PROTOCOL_VERSION,
10849                        manifest_concurrency(&initial),
10850                        ctx.egress.clone(),
10851                    )
10852                    .unwrap();
10853            } else {
10854                hello_via_sink(
10855                    &handler,
10856                    &ctx,
10857                    &mut rx,
10858                    hello_frame_with_manifest(initial.clone(), 1),
10859                )
10860                .await;
10861            }
10862            let response = handler
10863                .handle_control_frame(
10864                    &ctx,
10865                    catalog_update_with_capabilities_frame(
10866                        2,
10867                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
10868                            .capabilities
10869                            .unwrap(),
10870                    ),
10871                )
10872                .await
10873                .unwrap();
10874            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10875            assert_eq!(
10876                registry
10877                    .get_module_by_connection(conn)
10878                    .unwrap()
10879                    .unwrap()
10880                    .manifest,
10881                initial
10882            );
10883        }
10884    }
10885
10886    #[test]
10887    fn server_describe_surfaces_required_capability_verdict_fields() {
10888        let registry = Arc::new(Registry::default());
10889        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10890            [
10891                ("consumer".to_string(), true),
10892                ("provider".to_string(), false),
10893            ],
10894            BTreeMap::new(),
10895        );
10896        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10897        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10898            provides: Vec::new(),
10899            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10900                capability: "credentials-provider/v1".to_string(),
10901                need: subc_protocol::manifest::CapabilityNeed::Required,
10902            }],
10903            must_never_reach: Vec::new(),
10904        });
10905        let hello = Frame::build(
10906            FrameType::Hello,
10907            control_flags(),
10908            0,
10909            0,
10910            78,
10911            serde_json::to_vec(&ModuleHelloBody {
10912                manifest: consumer,
10913                protocol_ver: PROTOCOL_VERSION,
10914                control_ops: None,
10915                launch_nonce: None,
10916            })
10917            .expect("HELLO serializes"),
10918        )
10919        .expect("HELLO frame builds");
10920        handler
10921            .handle_control(ConnectionId::new(78), hello)
10922            .expect("consumer registers");
10923        let describe = Frame::build(
10924            FrameType::Request,
10925            control_flags(),
10926            0,
10927            0,
10928            79,
10929            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10930                .expect("request serializes"),
10931        )
10932        .expect("describe frame builds");
10933        let response = handler
10934            .handle_server_describe(describe)
10935            .expect("server.describe succeeds");
10936        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10937        let requirement = &rendered["capability_requirements"][0];
10938        assert_eq!(requirement["consumer"], "consumer");
10939        assert_eq!(requirement["verdict"], "never_provided");
10940        assert_eq!(requirement["episode_seq"], 1);
10941        assert_eq!(requirement["config_satisfiable"], false);
10942        assert_eq!(requirement["runtime_available"], false);
10943        assert!(requirement["detail"]
10944            .as_str()
10945            .expect("detail string")
10946            .contains("credentials-provider/v1"));
10947    }
10948
10949    /// Register a consumer that requires `project-identity/v1` and the provider
10950    /// of it, optionally mark daemon shutdown as begun, then deregister the
10951    /// provider as its connection closes. Returns the `capability.requirement`
10952    /// events that deregistration logged and the verdict the evaluator then
10953    /// holds for the requirement (the one `route.open` refusals read).
10954    fn deregister_required_provider(
10955        daemon_shutdown_begun: bool,
10956    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10957        const CAPABILITY: &str = "project-identity/v1";
10958        let registry = Arc::new(Registry::default());
10959        let daemon_shutdown = DaemonShutdownFlag::default();
10960        let handler = ControlHandler::new(Arc::clone(&registry))
10961            .with_capability_config(
10962                [
10963                    ("consumer".to_string(), true),
10964                    ("provider".to_string(), true),
10965                ],
10966                BTreeMap::new(),
10967            )
10968            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10969        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10970        consumer.capabilities = Some(CapabilityDeclarations {
10971            provides: Vec::new(),
10972            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10973                capability: CAPABILITY.to_string(),
10974                need: subc_protocol::manifest::CapabilityNeed::Required,
10975            }],
10976            must_never_reach: Vec::new(),
10977        });
10978        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10979        let provider_conn = ConnectionId::new(182);
10980        for (conn, manifest, corr) in [
10981            (ConnectionId::new(181), consumer, 181),
10982            (provider_conn, provider, 182),
10983        ] {
10984            let hello = handler
10985                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10986                .expect("module registers");
10987            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10988        }
10989        assert_eq!(
10990            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10991            Some(CapabilityVerdict::Provided),
10992            "the registered provider must satisfy the requirement first"
10993        );
10994        if daemon_shutdown_begun {
10995            daemon_shutdown.set_for_test();
10996        }
10997
10998        let capture = EventCapture::default();
10999        let _subscriber =
11000            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11001        handler
11002            .cleanup_connection(provider_conn)
11003            .expect("provider deregisters");
11004        let events = capture
11005            .events()
11006            .into_iter()
11007            .filter(|event| {
11008                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11009            })
11010            .collect();
11011        (
11012            events,
11013            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11014        )
11015    }
11016
11017    #[test]
11018    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11019        let (events, verdict) = deregister_required_provider(true);
11020        assert_eq!(
11021            verdict,
11022            Some(CapabilityVerdict::NeverProvided),
11023            "shutdown must not change the verdict route.open refusals read"
11024        );
11025        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11026        let event = &events[0];
11027        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11028        assert_eq!(
11029            event.fields.get("cause").map(String::as_str),
11030            Some("\"daemon_shutdown\""),
11031            "{event:?}"
11032        );
11033        assert_eq!(
11034            event.fields.get("verdict").map(String::as_str),
11035            Some("\"never_provided\""),
11036            "{event:?}"
11037        );
11038    }
11039
11040    #[test]
11041    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11042        let (events, verdict) = deregister_required_provider(false);
11043        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11044        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11045        let event = &events[0];
11046        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11047        assert_eq!(event.fields.get("cause"), None, "{event:?}");
11048        assert_eq!(
11049            event.fields.get("verdict").map(String::as_str),
11050            Some("\"never_provided\""),
11051            "{event:?}"
11052        );
11053    }
11054
11055    #[test]
11056    fn catalog_list_omits_capabilities_for_legacy_manifest() {
11057        let registry = Arc::new(Registry::default());
11058        let handler = ControlHandler::new(Arc::clone(&registry));
11059        let hello = handler
11060            .handle_control(
11061                ConnectionId::new(101),
11062                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11063            )
11064            .expect("legacy HELLO registers");
11065        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11066
11067        let request = Frame::build(
11068            FrameType::Request,
11069            control_flags(),
11070            0,
11071            0,
11072            102,
11073            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11074                .expect("catalog request serializes"),
11075        )
11076        .expect("catalog request frame builds");
11077        let response = handler
11078            .handle_catalog_list(request, None)
11079            .expect("catalog list succeeds");
11080        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11081        assert!(
11082            body["modules"][0].get("capabilities").is_none(),
11083            "legacy manifest must retain an absent capabilities field on catalog.list"
11084        );
11085    }
11086
11087    #[test]
11088    fn hello_ack_omits_storage_when_no_storage_config() {
11089        let registry = Arc::new(Registry::default());
11090        let handler = ControlHandler::new(Arc::clone(&registry));
11091        let responses = handler
11092            .handle_control(
11093                ConnectionId::new(1),
11094                hello_frame("aft", PROTOCOL_VERSION, 7),
11095            )
11096            .unwrap();
11097        let ack = parse_ack(&responses[0]);
11098        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11099        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11100    }
11101
11102    #[tokio::test]
11103    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11104        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11105        let registry = Arc::new(Registry::default());
11106        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
11107        let responses = handler
11108            .handle_control(
11109                ConnectionId::new(1),
11110                hello_frame("aft", PROTOCOL_VERSION, 7),
11111            )
11112            .unwrap();
11113        let ack = parse_ack(&responses[0]);
11114        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11115
11116        let described = handler
11117            .handle_control_frame(
11118                &route_ctx(ConnectionId::new(2)).0,
11119                Frame::build(
11120                    FrameType::Request,
11121                    control_flags(),
11122                    0,
11123                    0,
11124                    9,
11125                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11126                )
11127                .unwrap(),
11128            )
11129            .await
11130            .unwrap();
11131        let ClientControlResponse::ServerDescribe { machine_id, .. } =
11132            serde_json::from_slice(&described[0].body).unwrap()
11133        else {
11134            panic!("server.describe answered with another shape");
11135        };
11136        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11137    }
11138
11139    #[test]
11140    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11141        // With a central sqlite storage policy, each registering module gets its
11142        // own resolved descriptor in HELLO_ACK, keyed by its module id.
11143        let registry = Arc::new(Registry::default());
11144        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
11145            crate::daemon_config::StorageConfig::Sqlite {
11146                data_home: std::path::PathBuf::from("/data"),
11147            },
11148        ));
11149
11150        let responses = handler
11151            .handle_control(
11152                ConnectionId::new(1),
11153                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11154            )
11155            .unwrap();
11156        let ack = parse_ack(&responses[0]);
11157        assert_eq!(
11158            ack.storage,
11159            Some(serde_json::json!({
11160                "module_id": "alfonso-routing",
11161                "storage_namespace": "default",
11162                "isolation": { "kind": "module" },
11163                "backend": {
11164                    "backend": "sqlite",
11165                    "path": "/data/cortexkit/alfonso-routing/store.db"
11166                }
11167            })),
11168            "the delivered descriptor is the module's own sqlite store path"
11169        );
11170    }
11171
11172    #[test]
11173    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11174        let registry = Arc::new(Registry::default());
11175        let handler = ControlHandler::new(Arc::clone(&registry));
11176        let conn = ConnectionId::new(1);
11177        let responses = handler
11178            .handle_control(
11179                conn,
11180                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11181            )
11182            .unwrap();
11183        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11184        let registration = registry.get_module("aft").unwrap().unwrap();
11185        assert_eq!(registration.control_ops, module_baseline_control_ops());
11186
11187        let frame =
11188            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11189        assert!(handler
11190            .guard_module_control_op(&frame, "aft", "route.bind")
11191            .unwrap()
11192            .is_none());
11193        let error = handler
11194            .guard_module_control_op(&frame, "aft", "test.synthetic")
11195            .unwrap()
11196            .expect("synthetic ungranted op should be rejected");
11197        assert_eq!(error.header.ty, FrameType::Error);
11198        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11199    }
11200
11201    #[test]
11202    fn hello_control_ops_some_adds_optional_grants() {
11203        let registry = Arc::new(Registry::default());
11204        let handler = ControlHandler::new(Arc::clone(&registry));
11205        handler
11206            .handle_control(
11207                ConnectionId::new(1),
11208                hello_frame_with_control_ops(
11209                    "aft",
11210                    PROTOCOL_VERSION,
11211                    7,
11212                    Some(vec![
11213                        "future.synthetic".to_string(),
11214                        "route.bind".to_string(),
11215                    ]),
11216                ),
11217            )
11218            .unwrap();
11219        let registration = registry.get_module("aft").unwrap().unwrap();
11220        assert_eq!(
11221            registration.control_ops,
11222            vec![
11223                "route.bind".to_string(),
11224                "route.status".to_string(),
11225                "future.synthetic".to_string(),
11226            ]
11227        );
11228        let frame =
11229            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11230        assert!(handler
11231            .guard_module_control_op(&frame, "aft", "future.synthetic")
11232            .unwrap()
11233            .is_none());
11234    }
11235
11236    #[tokio::test]
11237    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11238        let registry = Arc::new(Registry::default());
11239        let forwarding = Arc::new(ForwardingTable::default());
11240        let handler =
11241            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11242        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11243        hello_via_sink(
11244            &handler,
11245            &module_ctx,
11246            &mut module_rx,
11247            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11248        )
11249        .await;
11250
11251        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11252        let responses = handler
11253            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11254            .await
11255            .unwrap();
11256        assert_eq!(responses.len(), 1);
11257        assert_eq!(responses[0].header.ty, FrameType::Error);
11258        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11259        assert!(module_rx.try_recv().is_err());
11260    }
11261
11262    #[tokio::test]
11263    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11264        let registry = Arc::new(Registry::default());
11265        let forwarding = Arc::new(ForwardingTable::default());
11266        let handler =
11267            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11268        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11269        hello_via_sink(
11270            &handler,
11271            &module_ctx,
11272            &mut module_rx,
11273            hello_frame_with_control_ops(
11274                "aft",
11275                PROTOCOL_VERSION,
11276                7,
11277                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11278            ),
11279        )
11280        .await;
11281
11282        let project_root = unique_project_root("demux");
11283        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11284        let route_handler = handler.clone();
11285        let route_task = tokio::spawn(async move {
11286            route_handler
11287                .handle_control_frame(
11288                    &route_client_ctx,
11289                    route_open_frame(100, "aft", project_root),
11290                )
11291                .await
11292                .unwrap()
11293        });
11294        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11295            .await
11296            .unwrap()
11297            .unwrap();
11298        assert!(matches!(
11299            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11300            ModuleControlRequest::RouteBind { .. }
11301        ));
11302
11303        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11304        let health_handler = handler.clone();
11305        let health_task = tokio::spawn(async move {
11306            health_handler
11307                .handle_control_frame(
11308                    &health_client_ctx,
11309                    supervisor_health_probe_frame(101, "aft"),
11310                )
11311                .await
11312                .unwrap()
11313        });
11314        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11315            .await
11316            .unwrap()
11317            .unwrap();
11318        assert_eq!(
11319            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11320            ModuleControlRequest::HealthCheck {}
11321        );
11322
11323        handler
11324            .handle_control_frame(
11325                &module_ctx,
11326                health_response(health_frame.header.corr, HealthStatus::Degraded),
11327            )
11328            .await
11329            .unwrap();
11330        let health_response = health_task.await.unwrap();
11331        assert_eq!(health_response.len(), 1);
11332        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11333            ClientControlResponse::SupervisorHealthProbe {
11334                module_id,
11335                status,
11336                detail,
11337                metrics,
11338            } => {
11339                assert_eq!(module_id, "aft");
11340                assert_eq!(status, HealthStatus::Degraded);
11341                assert_eq!(detail.as_deref(), Some("warming"));
11342                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11343            }
11344            other => panic!("unexpected health response: {other:?}"),
11345        }
11346
11347        handler
11348            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11349            .await
11350            .unwrap();
11351        let route_response = route_task.await.unwrap();
11352        assert!(route_response.is_empty());
11353        let published = route_client_rx.recv().await.unwrap();
11354        assert!(matches!(
11355            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11356            ClientControlResponse::RouteOpen { .. }
11357        ));
11358    }
11359
11360    /// Start one `route.open` on `client_connection` and return its still-running
11361    /// handler task together with the `route.bind` the module received for it.
11362    /// The handler blocks until the module answers, so it has to run as a task
11363    /// while the test drives the module side.
11364    async fn relay_route_open(
11365        handler: &ControlHandler,
11366        client_connection: ConnectionId,
11367        client_egress: &FrameSink,
11368        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11369        corr: u64,
11370        module_id: &str,
11371        project_root_label: &str,
11372    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11373        let ctx = RouteCtx {
11374            connection_id: client_connection,
11375            egress: client_egress.clone(),
11376        };
11377        let handler = handler.clone();
11378        let project_root = unique_project_root(project_root_label);
11379        let module_id = module_id.to_string();
11380        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11381        let task = tokio::spawn(async move {
11382            let _guard = tracing::dispatcher::set_default(&dispatch);
11383            handler
11384                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11385                .await
11386                .unwrap()
11387        });
11388        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11389            .await
11390            .expect("module receives the relayed route.bind")
11391            .expect("module egress is open");
11392        (task, bind.frame)
11393    }
11394
11395    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11396        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11397            ModuleControlRequest::RouteBind {
11398                route_channel,
11399                epoch,
11400                ..
11401            } => (route_channel, epoch),
11402            other => panic!("expected a route.bind request, got {other:?}"),
11403        }
11404    }
11405
11406    fn published_route(frame: &Frame) -> (u16, u32) {
11407        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11408            ClientControlResponse::RouteOpen {
11409                route_channel,
11410                route_epoch,
11411                ..
11412            } => (route_channel, route_epoch),
11413            other => panic!("expected a route.open response, got {other:?}"),
11414        }
11415    }
11416
11417    /// Reproduction of a production outage. A client had `route.open`s in
11418    /// flight to a module and was already marked closing -- its egress had refused a
11419    /// module frame, so the daemon asked its connection to end -- while its sink
11420    /// was still open. When the module acked those binds, the daemon refused to
11421    /// commit a route for a closing client, and that refusal was returned from
11422    /// the MODULE connection's frame handler, where a router error that has no
11423    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
11424    /// the supervisor correctly did not respawn a clean exit, and every seat lost
11425    /// its tools for hours -- one client's teardown took down a connection
11426    /// carrying ~170 other routes.
11427    ///
11428    /// The window is opened here by calling the production path that opens it
11429    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
11430    /// state that matters is "in `closing_connections`, sink still open, relay
11431    /// still pending", and it lasts only from the close request until the
11432    /// connection loop reacts to it; a socket-level test can flood a client into
11433    /// that escalation but cannot pin the module's ack inside the window. Closing
11434    /// the socket instead takes the other path entirely -- connection teardown
11435    /// removes the pending relay under the same lock, so the ack finds nothing.
11436    #[tokio::test]
11437    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11438        let registry = Arc::new(Registry::default());
11439        let forwarding = Arc::new(ForwardingTable::default());
11440        let handler =
11441            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11442
11443        let module_connection = ConnectionId::new(30);
11444        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11445        hello_via_sink(
11446            &handler,
11447            &module_ctx,
11448            &mut module_rx,
11449            hello_frame("aft", PROTOCOL_VERSION, 7),
11450        )
11451        .await;
11452
11453        let dying_client = ConnectionId::new(31);
11454        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11455
11456        // A published route on the dying client. The escalation below only marks
11457        // a connection closing for a route it has already published.
11458        let (first_task, first_bind) = relay_route_open(
11459            &handler,
11460            dying_client,
11461            &dying_ctx.egress,
11462            &mut module_rx,
11463            100,
11464            "aft",
11465            "closing-first",
11466        )
11467        .await;
11468        handler
11469            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11470            .await
11471            .unwrap();
11472        assert!(first_task.await.unwrap().is_empty());
11473        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11474
11475        // A second route.open from the same client, relayed and awaiting its ack.
11476        let (second_task, second_bind) = relay_route_open(
11477            &handler,
11478            dying_client,
11479            &dying_ctx.egress,
11480            &mut module_rx,
11481            101,
11482            "aft",
11483            "closing-second",
11484        )
11485        .await;
11486        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11487
11488        // The window: the client is closing, its sink is still open, and its
11489        // second bind is still pending.
11490        assert!(forwarding
11491            .escalate_client_delivery_failure(
11492                dying_client,
11493                first_channel,
11494                first_epoch,
11495                CloseReason::new(
11496                    "module_to_client_delivery_failed",
11497                    "client egress refused a module frame",
11498                ),
11499                crate::forwarding::UndeliveredFrame {
11500                    module_id: None,
11501                    sink: &dying_ctx.egress,
11502                },
11503            )
11504            .unwrap());
11505        assert!(!dying_ctx.egress.is_closed());
11506
11507        // The frame that used to end the module connection.
11508        let ack = handler
11509            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11510            .await;
11511        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11512        if module_loop_error.is_some() {
11513            // What the server's connection loop does with a router error that has
11514            // no ERROR-frame translation: end the connection, which releases the
11515            // module's registration and every route on it.
11516            handler.cleanup_connection(module_connection).unwrap();
11517        }
11518        // Read the module's next frame before opening the co-tenant's route, so
11519        // the GOODBYE assertion below is about THIS ack and not about later
11520        // traffic. `None` means the module was told nothing.
11521        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11522            .await
11523            .ok()
11524            .flatten();
11525
11526        // 1. The module connection is still registered.
11527        assert!(
11528            registry
11529                .get_module_by_connection(module_connection)
11530                .unwrap()
11531                .is_some(),
11532            "one client's closing connection ended the shared module connection: \
11533             {module_loop_error:?}"
11534        );
11535        // ...and still serving: another client can open and use a route on it.
11536        let cotenant = ConnectionId::new(32);
11537        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11538        let (cotenant_task, cotenant_bind) = relay_route_open(
11539            &handler,
11540            cotenant,
11541            &cotenant_ctx.egress,
11542            &mut module_rx,
11543            102,
11544            "aft",
11545            "closing-cotenant",
11546        )
11547        .await;
11548        handler
11549            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11550            .await
11551            .unwrap();
11552        assert!(cotenant_task.await.unwrap().is_empty());
11553        let (cotenant_channel, cotenant_epoch) =
11554            published_route(&cotenant_rx.recv().await.unwrap());
11555        assert!(matches!(
11556            forwarding
11557                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11558                .unwrap(),
11559            DataRoute::Client(DataRouteState::Bound(_))
11560        ));
11561
11562        // 2. The module was told to drop the binding it created for the route
11563        //    that will never be published.
11564        let goodbye = post_ack_module_frame
11565            .expect("module receives a GOODBYE for the abandoned route channel");
11566        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11567        assert_eq!(goodbye.header.channel, abandoned_channel);
11568        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11569
11570        // 3. The dying client received nothing: no route was ever published to
11571        //    it. Its route.open is answered as unavailable, which the connection
11572        //    loop would write to a socket that is already going away.
11573        assert!(dying_rx.try_recv().is_err());
11574        let second_response = second_task.await.unwrap();
11575        assert_eq!(second_response.len(), 1);
11576        assert_eq!(
11577            parse_error(&second_response[0])["code"],
11578            "target_unavailable"
11579        );
11580    }
11581
11582    /// One `catalog.list` as a client sees it: the generation and the listed
11583    /// module ids with their readiness.
11584    async fn catalog_snapshot(
11585        handler: &ControlHandler,
11586        client_ctx: &RouteCtx,
11587        corr: u64,
11588    ) -> (u64, Vec<(String, bool)>) {
11589        let body =
11590            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11591        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11592        let reply = handler
11593            .handle_control_frame(client_ctx, request)
11594            .await
11595            .unwrap()
11596            .pop()
11597            .unwrap();
11598        let ClientControlResponse::CatalogList {
11599            generation,
11600            modules,
11601            ..
11602        } = serde_json::from_slice(&reply.body).unwrap()
11603        else {
11604            panic!("not a catalog.list reply");
11605        };
11606        (
11607            generation,
11608            modules
11609                .into_iter()
11610                .map(|entry| (entry.module_id, entry.ready))
11611                .collect(),
11612        )
11613    }
11614
11615    /// A handler whose HELLO for module "aft" on connection 40 is parked
11616    /// between reserving its registry entry and installing its forwarding
11617    /// endpoint. Returns the handler, the two-party barrier (wait once to know
11618    /// the HELLO is parked, once more to release it), the HELLO's thread, and
11619    /// the module's context and receiver.
11620    fn hello_held_before_forwarding() -> (
11621        ControlHandler,
11622        Arc<std::sync::Barrier>,
11623        std::thread::JoinHandle<Vec<Frame>>,
11624        RouteCtx,
11625        mpsc::Receiver<crate::router::OutboundFrame>,
11626    ) {
11627        let hold = Arc::new(std::sync::Barrier::new(2));
11628        let handler = ControlHandler::with_forwarding(
11629            Arc::new(Registry::default()),
11630            Arc::new(ForwardingTable::default()),
11631        )
11632        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11633        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11634        let hello = {
11635            let handler = handler.clone();
11636            let module_ctx = module_ctx.clone();
11637            std::thread::spawn(move || {
11638                handler
11639                    .handle_hello(
11640                        module_ctx.connection_id,
11641                        Some(module_ctx.egress.clone()),
11642                        hello_frame("aft", PROTOCOL_VERSION, 7),
11643                    )
11644                    .unwrap()
11645            })
11646        };
11647        (handler, hold, hello, module_ctx, module_rx)
11648    }
11649
11650    /// While a HELLO has reserved its registry entry but its forwarding
11651    /// endpoint is not installed yet, `route.open` to the module can only be
11652    /// refused. `catalog.list` must not list the module in that window: a
11653    /// client that opens a route as soon as it sees a module listed and ready
11654    /// would be refused for a module the daemon had just advertised. The
11655    /// refusal in the window stays the retryable `target_unavailable`, never
11656    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
11657    /// Once the HELLO completes, the module is listed and a route to it opens.
11658    #[tokio::test]
11659    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11660        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11661        let client = ConnectionId::new(41);
11662        let (client_ctx, mut client_rx) = route_ctx(client);
11663
11664        hold.wait();
11665        // Parked: the registry holds "aft", forwarding has no endpoint for it.
11666        assert!(handler.registry.get_module("aft").unwrap().is_some());
11667        assert!(!handler
11668            .forwarding
11669            .has_live_module_connection("aft")
11670            .unwrap());
11671        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11672        assert!(
11673            listed.is_empty(),
11674            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11675        );
11676        let refused = handler
11677            .handle_control_frame(
11678                &client_ctx,
11679                route_open_frame(301, "aft", unique_project_root("listed-window")),
11680            )
11681            .await
11682            .unwrap();
11683        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11684        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11685        let duplicate = handler
11686            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11687            .await
11688            .unwrap();
11689        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11690
11691        hold.wait();
11692        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11693        let ack = module_rx.try_recv().unwrap().frame;
11694        assert_eq!(ack.header.ty, FrameType::HelloAck);
11695        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11696        assert_eq!(listed, vec![("aft".to_string(), true)]);
11697
11698        let (open_task, bind) = relay_route_open(
11699            &handler,
11700            client,
11701            &client_ctx.egress,
11702            &mut module_rx,
11703            303,
11704            "aft",
11705            "listed-after",
11706        )
11707        .await;
11708        assert!(handler
11709            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11710            .await
11711            .unwrap()
11712            .is_empty());
11713        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11714        published_route(&client_rx.try_recv().unwrap().frame);
11715    }
11716
11717    /// A catalog watcher keys on the generation: when it changes, it re-reads
11718    /// the listing. So the generation must advance exactly when the module
11719    /// appears. If it advanced while the module was still hidden, a watcher
11720    /// would read the new generation without the module, and the module would
11721    /// later appear under that same generation with nothing telling the
11722    /// watcher to look again.
11723    #[tokio::test]
11724    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11725        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11726        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11727        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11728
11729        hold.wait();
11730        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11731        assert!(listed.is_empty());
11732        assert_eq!(
11733            during, before,
11734            "the generation moved while the module was not listed"
11735        );
11736
11737        hold.wait();
11738        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11739        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11740        assert_eq!(listed, vec![("aft".to_string(), true)]);
11741        assert_eq!(
11742            after,
11743            before + 1,
11744            "the module appeared without exactly one generation bump"
11745        );
11746    }
11747
11748    /// A HELLO whose forwarding endpoint cannot be installed (here the
11749    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11750    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11751    /// and the catalog generation where it was.
11752    #[tokio::test]
11753    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11754        let registry = Arc::new(Registry::default());
11755        let handler = ControlHandler::with_forwarding(
11756            Arc::clone(&registry),
11757            Arc::new(ForwardingTable::default()),
11758        );
11759        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11760        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11761        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11762        drop(module_rx);
11763
11764        let replies = handler
11765            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11766            .await
11767            .unwrap();
11768        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11769        assert!(registry.get_module("aft").unwrap().is_none());
11770        assert!(registry
11771            .get_module_by_connection(module_ctx.connection_id)
11772            .unwrap()
11773            .is_none());
11774        assert_eq!(
11775            catalog_snapshot(&handler, &client_ctx, 301).await,
11776            (before, Vec::new())
11777        );
11778    }
11779
11780    /// A handler whose `route.open` parks after queueing the route.bind and
11781    /// before marking it enqueued, with module "aft" registered on connection
11782    /// 40. Returns the two-party barrier that parks and releases it.
11783    async fn route_open_held_after_bind_send() -> (
11784        ControlHandler,
11785        Arc<tokio::sync::Barrier>,
11786        RouteCtx,
11787        mpsc::Receiver<crate::router::OutboundFrame>,
11788    ) {
11789        let hold = Arc::new(tokio::sync::Barrier::new(2));
11790        let handler = ControlHandler::with_forwarding(
11791            Arc::new(Registry::default()),
11792            Arc::new(ForwardingTable::default()),
11793        )
11794        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11795        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11796        hello_via_sink(
11797            &handler,
11798            &module_ctx,
11799            &mut module_rx,
11800            hello_frame("aft", PROTOCOL_VERSION, 7),
11801        )
11802        .await;
11803        (handler, hold, module_ctx, module_rx)
11804    }
11805
11806    /// A fast module acks the route.bind, and the reader commits the route,
11807    /// before `route.open` has marked the relay enqueued. The daemon used to
11808    /// read the missing relay as an abandoned bind and send the module a
11809    /// GOODBYE for the route it had just committed: the client was told its
11810    /// route was open, the module unbound the channel, and the client's first
11811    /// request was dropped on the module side as unbound.
11812    #[tokio::test]
11813    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11814        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11815        let client = ConnectionId::new(41);
11816        let (client_ctx, mut client_rx) = route_ctx(client);
11817
11818        let (open_task, bind) = relay_route_open(
11819            &handler,
11820            client,
11821            &client_ctx.egress,
11822            &mut module_rx,
11823            200,
11824            "aft",
11825            "acked-before-mark",
11826        )
11827        .await;
11828        let (module_channel, module_epoch) = route_bind_channel(&bind);
11829        // route.open is now parked between queueing the bind and marking it.
11830        hold.wait().await;
11831
11832        // The module answers inside that window, and the route commits.
11833        assert!(handler
11834            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11835            .await
11836            .unwrap()
11837            .is_empty());
11838        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11839
11840        hold.wait().await;
11841        assert!(
11842            open_task.await.unwrap().is_empty(),
11843            "route.open answers accepted"
11844        );
11845
11846        // Every frame route.open could have sent is already queued, because it
11847        // has returned; the module must have been sent nothing.
11848        match module_rx.try_recv() {
11849            Err(mpsc::error::TryRecvError::Empty) => {}
11850            Ok(outbound) => panic!(
11851                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11852                outbound.frame.header.ty,
11853                outbound.frame.header.channel,
11854                outbound.frame.header.epoch
11855            ),
11856            Err(err) => panic!("module egress closed: {err}"),
11857        }
11858
11859        // The client's first request on the route is the next thing the module
11860        // reads, on the binding it just created.
11861        let router = Router::with_control_handler(Arc::new(handler.clone()));
11862        let request = Frame::build(
11863            FrameType::Request,
11864            Flags::new(true, Priority::Interactive, false),
11865            client_channel,
11866            client_epoch,
11867            201,
11868            b"first request".to_vec(),
11869        )
11870        .unwrap();
11871        router
11872            .route_for_connection(&client_ctx, request)
11873            .await
11874            .unwrap();
11875        let delivered = module_rx.try_recv().unwrap().frame;
11876        assert_eq!(delivered.header.ty, FrameType::Request);
11877        assert_eq!(
11878            (delivered.header.channel, delivered.header.epoch),
11879            (module_channel, module_epoch)
11880        );
11881        assert_eq!(delivered.body, b"first request");
11882    }
11883
11884    /// The client goes away while `route.open` is still queueing the bind, so
11885    /// the relay is dropped with no module reply. The teardown cannot send the
11886    /// GOODBYE itself (it could reach the module ahead of the bind), so
11887    /// `route.open` must send it once the bind is queued; otherwise the module
11888    /// keeps a binding the daemon will never commit.
11889    #[tokio::test]
11890    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11891        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11892        let client = ConnectionId::new(42);
11893        let (client_ctx, _client_rx) = route_ctx(client);
11894
11895        let (open_task, bind) = relay_route_open(
11896            &handler,
11897            client,
11898            &client_ctx.egress,
11899            &mut module_rx,
11900            300,
11901            "aft",
11902            "released-before-mark",
11903        )
11904        .await;
11905        let (module_channel, module_epoch) = route_bind_channel(&bind);
11906        hold.wait().await;
11907
11908        handler.cleanup_connection(client).unwrap();
11909        assert!(
11910            module_rx.try_recv().is_err(),
11911            "client teardown must not GOODBYE a bind route.open is still queueing"
11912        );
11913
11914        hold.wait().await;
11915        let response = open_task.await.unwrap();
11916        assert_eq!(response.len(), 1);
11917        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11918
11919        let goodbye = module_rx
11920            .try_recv()
11921            .expect("module is told to drop the binding the daemon abandoned")
11922            .frame;
11923        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11924        assert_eq!(
11925            (goodbye.header.channel, goodbye.header.epoch),
11926            (module_channel, module_epoch)
11927        );
11928        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11929    }
11930
11931    /// The module acks inside the same window, but the client's egress closed
11932    /// first, so the reader refuses to commit. The ack proves the module holds
11933    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11934    /// send a second one.
11935    #[tokio::test]
11936    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11937        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11938        let client = ConnectionId::new(43);
11939        let (client_ctx, client_rx) = route_ctx(client);
11940
11941        let (open_task, bind) = relay_route_open(
11942            &handler,
11943            client,
11944            &client_ctx.egress,
11945            &mut module_rx,
11946            400,
11947            "aft",
11948            "gone-client-before-mark",
11949        )
11950        .await;
11951        let (module_channel, module_epoch) = route_bind_channel(&bind);
11952        hold.wait().await;
11953
11954        drop(client_rx);
11955        assert!(client_ctx.egress.is_closed());
11956        handler
11957            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11958            .await
11959            .unwrap();
11960
11961        hold.wait().await;
11962        let response = open_task.await.unwrap();
11963        assert_eq!(response.len(), 1);
11964        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11965
11966        let goodbye = module_rx
11967            .try_recv()
11968            .expect("module is told to drop the binding it acked")
11969            .frame;
11970        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11971        assert_eq!(
11972            (goodbye.header.channel, goodbye.header.epoch),
11973            (module_channel, module_epoch)
11974        );
11975        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11976        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11977    }
11978
11979    /// The fence at the module-loop boundary, stated as its own contract: which
11980    /// forwarding failures are allowed to end the module connection that is being
11981    /// served. A `ConnectionClosing` naming some client is about that client, and
11982    /// a module connection is shared; the same error naming the module's own
11983    /// connection is about this connection and must stay fatal, as must failures
11984    /// that are about the forwarding table itself.
11985    #[test]
11986    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11987        let handler = ControlHandler::default();
11988        let module_connection = ConnectionId::new(30);
11989        let client_connection = ConnectionId::new(31);
11990
11991        handler
11992            .refuse_to_end_module_connection_for_a_client(
11993                module_connection,
11994                77,
11995                ForwardingError::ConnectionClosing {
11996                    connection_id: client_connection,
11997                },
11998            )
11999            .expect("a closing client must never end the module connection");
12000
12001        assert!(matches!(
12002            handler.refuse_to_end_module_connection_for_a_client(
12003                module_connection,
12004                78,
12005                ForwardingError::ConnectionClosing {
12006                    connection_id: module_connection,
12007                },
12008            ),
12009            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12010                connection_id
12011            })) if connection_id == module_connection
12012        ));
12013        assert!(matches!(
12014            handler.refuse_to_end_module_connection_for_a_client(
12015                module_connection,
12016                79,
12017                ForwardingError::Poisoned,
12018            ),
12019            Err(RouterError::Forwarding(ForwardingError::Poisoned))
12020        ));
12021        assert!(matches!(
12022            handler.refuse_to_end_module_connection_for_a_client(
12023                module_connection,
12024                80,
12025                ForwardingError::StaleModuleEndpoint,
12026            ),
12027            Err(RouterError::Forwarding(
12028                ForwardingError::StaleModuleEndpoint
12029            ))
12030        ));
12031    }
12032
12033    /// The spawn-attestation guard is what stops a connected module from claiming
12034    /// another module's identity and being stamped `Reserved` for it. Every other
12035    /// test that supplies a consumer_identity supplies a CORRECT one, because a
12036    /// correct one is what the rest of the flow needs -- so the guard's rejection
12037    /// branch was never the subject of an assertion, only its acceptance branch.
12038    ///
12039    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
12040    /// whole subc-core library suite green; only the forwarding integration tests
12041    /// notice, and they notice for unrelated reasons. This test exists so the
12042    /// refusal itself is asserted where the guard lives: it fails if the identity
12043    /// check stops refusing, which is the direction that matters, since a guard
12044    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
12045    #[tokio::test]
12046    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12047        let registry = Arc::new(Registry::default());
12048        let forwarding = Arc::new(ForwardingTable::default());
12049        let supervisor = SupervisorHandle::new();
12050        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12051        let handler =
12052            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12053                .with_supervisor(supervisor);
12054
12055        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12056        hello_via_sink(
12057            &handler,
12058            &target_ctx,
12059            &mut target_rx,
12060            hello_frame("target", PROTOCOL_VERSION, 1),
12061        )
12062        .await;
12063
12064        // A real supervised module id presenting the wrong nonce. This is the
12065        // impersonation case: the attacker knows a privileged module_id, which is
12066        // public, and guesses at the nonce, which is not.
12067        let wrong_nonce = handler
12068            .handle_control_frame(
12069                &route_ctx(ConnectionId::new(91)).0,
12070                route_open_frame_with_admission_facts(
12071                    20,
12072                    "target",
12073                    unique_project_root("admission-facts"),
12074                    Some(subc_control::ConsumerIdentity {
12075                        module_id: "fed".to_string(),
12076                        launch_nonce: "not-the-real-nonce".to_string(),
12077                    }),
12078                    None,
12079                ),
12080            )
12081            .await
12082            .unwrap();
12083        assert_eq!(
12084            parse_error(&wrong_nonce[0])["code"],
12085            "bad_consumer_identity",
12086            "a mismatched launch nonce must be refused, not stamped Reserved"
12087        );
12088
12089        // A module id the supervisor never spawned at all, so no nonce exists to
12090        // compare against. An implementation that treats "no record" as "nothing
12091        // to check" fails open here while passing the case above.
12092        let never_spawned = handler
12093            .handle_control_frame(
12094                &route_ctx(ConnectionId::new(92)).0,
12095                route_open_frame_with_admission_facts(
12096                    21,
12097                    "target",
12098                    unique_project_root("admission-facts"),
12099                    Some(subc_control::ConsumerIdentity {
12100                        module_id: "never-spawned".to_string(),
12101                        launch_nonce: "any-nonce".to_string(),
12102                    }),
12103                    None,
12104                ),
12105            )
12106            .await
12107            .unwrap();
12108        assert_eq!(
12109            parse_error(&never_spawned[0])["code"],
12110            "bad_consumer_identity",
12111            "an unspawned module_id must be refused rather than accepted for lack of a record"
12112        );
12113    }
12114
12115    /// The refusal test above proves the guard says NO. Nothing proved it can say
12116    /// YES, and the difference is not academic: replacing the whole authorization
12117    /// with `false` -- admitting no consumer identity at all, revoking Reserved
12118    /// standing for every supervised module in the fleet -- leaves 110 of the 111
12119    /// library tests GREEN. The one that notices does so by HANGING, because it
12120    /// waits for a bind that can no longer happen.
12121    ///
12122    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
12123    /// or flaky test, invites a RETRY rather than an investigation, and the retry
12124    /// hangs too and gets blamed on the runner. So a total revocation of the
12125    /// daemon's trust grant would have shipped behind a symptom nobody attributes
12126    /// to code.
12127    ///
12128    /// The bias is structural rather than accidental. A REFUSAL looks like a
12129    /// failure someone writes a test for; a GRANT looks like the happy path. Every
12130    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
12131    /// suite for that reason, and this one is the purest case in the daemon.
12132    ///
12133    /// This test asserts the EFFECT rather than the absence of an error: the module
12134    /// receives a RouteBind and it carries `Reserved` naming the attested module.
12135    /// A guard that admitted nobody would produce no bind at all; one that admitted
12136    /// everybody would stamp the wrong principal, which the refusal test catches.
12137    #[tokio::test]
12138    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12139        let registry = Arc::new(Registry::default());
12140        let forwarding = Arc::new(ForwardingTable::default());
12141        let supervisor = SupervisorHandle::new();
12142        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12143        let handler =
12144            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12145                .with_supervisor(supervisor);
12146
12147        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12148        hello_via_sink(
12149            &handler,
12150            &target_ctx,
12151            &mut target_rx,
12152            hello_frame("target", PROTOCOL_VERSION, 1),
12153        )
12154        .await;
12155
12156        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12157        let route_handler = handler.clone();
12158        let route_task = tokio::spawn(async move {
12159            route_handler
12160                .handle_control_frame(
12161                    &client_ctx,
12162                    route_open_frame_with_admission_facts(
12163                        30,
12164                        "target",
12165                        unique_project_root("admission-facts"),
12166                        Some(subc_control::ConsumerIdentity {
12167                            module_id: "fed".to_string(),
12168                            launch_nonce: "fed-nonce".to_string(),
12169                        }),
12170                        None,
12171                    ),
12172                )
12173                .await
12174                .unwrap()
12175        });
12176
12177        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
12178        // the very mutation it exists to catch -- a guard that admits nobody -- no
12179        // bind is ever sent, so it HUNG rather than failing. That reproduces the
12180        // exact defect being fixed: a total revocation detected only as a stalled
12181        // suite, which reads as flakiness and invites a retry. An acceptance test
12182        // that waits for an effect must bound the wait, or a red becomes a hang.
12183        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12184            .await
12185            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12186            .expect("module control channel closed before route.bind");
12187        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12188        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12189            panic!("expected route.bind")
12190        };
12191        assert_eq!(
12192            principal,
12193            Some(Principal::Reserved {
12194                module_id: "fed".to_string()
12195            }),
12196            "a correctly attested consumer must be stamped Reserved for its own id"
12197        );
12198
12199        handler
12200            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12201            .await
12202            .unwrap();
12203        assert!(route_task.await.unwrap().is_empty());
12204        assert!(
12205            matches!(
12206                serde_json::from_slice::<ClientControlResponse>(
12207                    &client_rx.recv().await.unwrap().body
12208                )
12209                .unwrap(),
12210                ClientControlResponse::RouteOpen { .. }
12211            ),
12212            "the route must actually open, not merely avoid an error"
12213        );
12214    }
12215
12216    #[tokio::test(start_paused = true)]
12217    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12218        let registry = Arc::new(Registry::default());
12219        let forwarding = Arc::new(ForwardingTable::default());
12220        let supervisor = SupervisorHandle::new();
12221        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12222        let handler =
12223            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12224                .with_supervisor(supervisor);
12225
12226        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12227        hello_via_sink(
12228            &handler,
12229            &target_ctx,
12230            &mut target_rx,
12231            hello_frame("target", PROTOCOL_VERSION, 1),
12232        )
12233        .await;
12234
12235        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12236        let direct_handler = handler.clone();
12237        let direct_open = tokio::spawn(async move {
12238            direct_handler
12239                .handle_control_frame(
12240                    &direct_ctx,
12241                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
12242                )
12243                .await
12244                .unwrap()
12245        });
12246        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12247            .await
12248            .expect("no direct route.bind within 5s")
12249            .expect("target control channel closed before direct route.bind");
12250        handler
12251            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12252            .await
12253            .unwrap();
12254        assert!(direct_open.await.unwrap().is_empty());
12255        let _ = direct_rx.recv().await.unwrap();
12256
12257        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12258        let reserved_handler = handler.clone();
12259        let reserved_open = tokio::spawn(async move {
12260            reserved_handler
12261                .handle_control_frame(
12262                    &reserved_ctx,
12263                    route_open_frame_with_admission_facts(
12264                        3,
12265                        "target",
12266                        unique_project_root("admission-facts"),
12267                        Some(ConsumerIdentity {
12268                            module_id: "fed".to_string(),
12269                            launch_nonce: "fed-nonce".to_string(),
12270                        }),
12271                        None,
12272                    ),
12273                )
12274                .await
12275                .unwrap()
12276        });
12277        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12278            .await
12279            .expect("no reserved route.bind within 5s")
12280            .expect("target control channel closed before reserved route.bind");
12281        handler
12282            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12283            .await
12284            .unwrap();
12285        assert!(reserved_open.await.unwrap().is_empty());
12286        let _ = reserved_rx.recv().await.unwrap();
12287
12288        forwarding
12289            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12290            .unwrap();
12291        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12292        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12293            module_id: Some("target".to_string()),
12294        })
12295        .unwrap();
12296        let census_frame =
12297            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12298        let response = handler
12299            .handle_control_frame(&census_ctx, census_frame)
12300            .await
12301            .unwrap()
12302            .pop()
12303            .unwrap();
12304        let actual: Value = serde_json::from_slice(&response.body).unwrap();
12305        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12306        assert!(matches!(
12307            decoded,
12308            ClientControlResponse::SupervisorRoutes { .. }
12309        ));
12310        let routes = actual["modules"][0]["routes"].as_array().unwrap();
12311        assert_eq!(routes.len(), 2);
12312        assert!(routes.iter().all(|route| route["draining"] == true));
12313        // The census carries WHY: the reason the drain was begun with, in the
12314        // route.closing vocabulary, on every draining route this drain marked.
12315        assert!(
12316            routes.iter().all(|route| route["drain_reason"] == "reload"),
12317            "draining routes must name the drain's reason: {routes:?}"
12318        );
12319        assert!(routes.iter().any(|route| {
12320            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12321        }));
12322        assert!(routes.iter().any(|route| {
12323            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12324        }));
12325
12326        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12327            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12328        if std::env::var_os("UPDATE_GOLDEN").is_some() {
12329            std::fs::write(
12330                &golden_path,
12331                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12332            )
12333            .unwrap();
12334        }
12335        let expected: Value =
12336            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12337        assert_eq!(actual, expected);
12338    }
12339
12340    async fn query_live_roots(
12341        handler: &ControlHandler,
12342        module_ctx: &RouteCtx,
12343    ) -> ModuleControlResponseToModule {
12344        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12345        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12346        let response = handler
12347            .handle_control_frame(module_ctx, frame)
12348            .await
12349            .unwrap()
12350            .pop()
12351            .unwrap();
12352        serde_json::from_slice(&response.body).unwrap()
12353    }
12354
12355    #[tokio::test(start_paused = true)]
12356    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12357        let registry = Arc::new(Registry::default());
12358        let forwarding = Arc::new(ForwardingTable::default());
12359        let handler = ControlHandler::with_forwarding(registry, forwarding);
12360        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12361        hello_via_sink(
12362            &handler,
12363            &target_ctx,
12364            &mut target_rx,
12365            hello_frame("target", PROTOCOL_VERSION, 1),
12366        )
12367        .await;
12368        let root = unique_project_root("live-roots-known");
12369        let path = ProjectRootId::from_path_allowing_missing(root.path())
12370            .unwrap()
12371            .as_path()
12372            .to_path_buf();
12373        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12374        let open_handler = handler.clone();
12375        let opened = tokio::spawn(async move {
12376            open_handler
12377                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12378                .await
12379                .unwrap()
12380        });
12381        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12382            .await
12383            .unwrap()
12384            .unwrap();
12385        handler
12386            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12387            .await
12388            .unwrap();
12389        assert!(opened.await.unwrap().is_empty());
12390        let _ = client_rx.recv().await.unwrap();
12391
12392        let root = unique_project_root("live-roots-pending");
12393        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12394            .unwrap()
12395            .as_path()
12396            .to_path_buf();
12397        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12398        let open_handler = handler.clone();
12399        let pending = tokio::spawn(async move {
12400            open_handler
12401                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12402                .await
12403                .unwrap()
12404        });
12405        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12406            .await
12407            .unwrap()
12408            .unwrap();
12409        let actual = query_live_roots(&handler, &target_ctx).await;
12410        let ModuleControlResponseToModule::LiveRoots {
12411            roots,
12412            unknown_root_bindings,
12413            total_bindings,
12414        } = actual
12415        else {
12416            panic!("expected live roots")
12417        };
12418        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12419        assert_eq!(unknown_root_bindings, 0);
12420        assert_eq!(
12421            roots.len(),
12422            2,
12423            "root-known arm must retain each canonical root"
12424        );
12425        assert_eq!(
12426            total_bindings,
12427            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12428        );
12429        let counts = roots
12430            .iter()
12431            .map(|root| (root.project_root.clone(), root.bound, root.pending))
12432            .collect::<Vec<_>>();
12433        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12434        expected.sort_by(|a, b| a.0.cmp(&b.0));
12435        assert_eq!(
12436            counts, expected,
12437            "roots must sort by path and count pending separately"
12438        );
12439        handler
12440            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12441            .await
12442            .unwrap();
12443        assert!(pending.await.unwrap().is_empty());
12444    }
12445
12446    #[tokio::test(start_paused = true)]
12447    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12448        let forwarding = Arc::new(ForwardingTable::default());
12449        let handler =
12450            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12451        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12452        hello_via_sink(
12453            &handler,
12454            &target_ctx,
12455            &mut target_rx,
12456            hello_frame("target", PROTOCOL_VERSION, 1),
12457        )
12458        .await;
12459        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12460        let pending = forwarding
12461            .begin_route_bind_relay_for_test(
12462                client_ctx.connection_id,
12463                client_ctx.egress.clone(),
12464                2,
12465                "target",
12466            )
12467            .unwrap();
12468        forwarding
12469            .complete_pending_relay(
12470                target_ctx.connection_id,
12471                pending.corr,
12472                RouteBindRelayOutcome::Accepted,
12473            )
12474            .unwrap();
12475        let actual = query_live_roots(&handler, &target_ctx).await;
12476        let ModuleControlResponseToModule::LiveRoots {
12477            roots,
12478            unknown_root_bindings,
12479            total_bindings,
12480        } = actual
12481        else {
12482            panic!("expected live roots")
12483        };
12484        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12485        assert_eq!(
12486            unknown_root_bindings, 1,
12487            "unknown-root arm must not read as no bindings"
12488        );
12489        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12490        assert_eq!(
12491            total_bindings,
12492            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12493        );
12494    }
12495
12496    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12497    /// so the ack has to be on its outbound queue before the module is
12498    /// routable. The connection loop writes a handler's replies only after the
12499    /// handler returns; this test stops in exactly that gap, runs a real
12500    /// route.open from another connection, and only then writes whatever the
12501    /// HELLO handler returned, the way the loop would. If the ack were still a
12502    /// reply, the route.bind request would reach the module first.
12503    #[tokio::test(start_paused = true)]
12504    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12505        let forwarding = Arc::new(ForwardingTable::default());
12506        let handler =
12507            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12508        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12509        let replies = handler
12510            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12511            .await
12512            .unwrap();
12513        let queued_by_hello = module_rx.len();
12514
12515        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12516        let open_handler = handler.clone();
12517        let open = tokio::spawn(async move {
12518            open_handler
12519                .handle_control_frame(
12520                    &client_ctx,
12521                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12522                )
12523                .await
12524                .unwrap()
12525        });
12526        // Let the route.open run until its route.bind is on the module's queue.
12527        let mut spins = 0;
12528        while module_rx.len() == queued_by_hello {
12529            spins += 1;
12530            assert!(spins < 10_000, "route.open never queued a route.bind");
12531            tokio::task::yield_now().await;
12532        }
12533
12534        // Now the connection loop's half: write the HELLO handler's replies.
12535        for reply in replies {
12536            module_ctx.egress.send(reply).await.unwrap();
12537        }
12538
12539        let first = module_rx.recv().await.unwrap().frame;
12540        assert_eq!(
12541            first.header.ty,
12542            FrameType::HelloAck,
12543            "the first frame a registering module reads must be its HELLO_ACK"
12544        );
12545        assert_eq!(first.header.corr, 7);
12546        let second = module_rx.recv().await.unwrap().frame;
12547        assert_eq!(second.header.ty, FrameType::Request);
12548        assert!(
12549            matches!(
12550                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12551                ModuleControlRequest::RouteBind { .. }
12552            ),
12553            "the route.bind follows the ack"
12554        );
12555        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12556
12557        handler
12558            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12559            .await
12560            .unwrap();
12561        assert!(open.await.unwrap().is_empty());
12562        let _ = client_rx.recv().await.unwrap();
12563    }
12564
12565    #[tokio::test(start_paused = true)]
12566    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12567        let handler = ControlHandler::with_forwarding(
12568            Arc::new(Registry::default()),
12569            Arc::new(ForwardingTable::default()),
12570        );
12571        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12572        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12573        hello_via_sink(
12574            &handler,
12575            &first_ctx,
12576            &mut first_rx,
12577            hello_frame("first", PROTOCOL_VERSION, 1),
12578        )
12579        .await;
12580        hello_via_sink(
12581            &handler,
12582            &second_ctx,
12583            &mut second_rx,
12584            hello_frame("second", PROTOCOL_VERSION, 2),
12585        )
12586        .await;
12587        let root = unique_project_root("second-only");
12588        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12589        let cloned = handler.clone();
12590        let open = tokio::spawn(async move {
12591            cloned
12592                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12593                .await
12594                .unwrap()
12595        });
12596        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12597            .await
12598            .unwrap()
12599            .unwrap();
12600        let first = query_live_roots(&handler, &first_ctx).await;
12601        let second = query_live_roots(&handler, &second_ctx).await;
12602        assert!(
12603            matches!(
12604                first,
12605                ModuleControlResponseToModule::LiveRoots {
12606                    total_bindings: 0,
12607                    ..
12608                }
12609            ),
12610            "cross-module scope must not expose another module's roots"
12611        );
12612        assert!(
12613            matches!(
12614                second,
12615                ModuleControlResponseToModule::LiveRoots {
12616                    total_bindings: 1,
12617                    ..
12618                }
12619            ),
12620            "second module must see its pending route"
12621        );
12622        handler
12623            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12624            .await
12625            .unwrap();
12626        assert!(open.await.unwrap().is_empty());
12627    }
12628
12629    #[tokio::test(start_paused = true)]
12630    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12631        let handler = ControlHandler::with_forwarding(
12632            Arc::new(Registry::default()),
12633            Arc::new(ForwardingTable::default()),
12634        );
12635        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12636        hello_via_sink(
12637            &handler,
12638            &target_ctx,
12639            &mut target_rx,
12640            hello_frame("target", PROTOCOL_VERSION, 1),
12641        )
12642        .await;
12643        let actual = query_live_roots(&handler, &target_ctx).await;
12644        let ModuleControlResponseToModule::LiveRoots {
12645            roots,
12646            unknown_root_bindings,
12647            total_bindings,
12648        } = actual
12649        else {
12650            panic!("expected live roots")
12651        };
12652        assert!(roots.is_empty());
12653        assert_eq!(unknown_root_bindings, 0);
12654        assert_eq!(total_bindings, 0);
12655        assert_eq!(
12656            total_bindings,
12657            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12658        );
12659    }
12660
12661    /// Read the vendored fed corpus rather than hand-building a package.
12662    ///
12663    /// A hand-built object encodes what the test author believed the carrier
12664    /// emits. These vectors are what it actually emits, and one of them exists
12665    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
12666    /// ignores additive unknown fields at the traversal emit terminus."
12667    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12668        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12669            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12670        let text = std::fs::read_to_string(&path)
12671            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12672        let vectors: Vec<(String, Value)> = text
12673            .lines()
12674            .filter(|line| !line.trim().is_empty())
12675            .map(|line| {
12676                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12677                let id = entry["corpus_id"]
12678                    .as_str()
12679                    .expect("every vector carries a corpus_id")
12680                    .to_string();
12681                (id, entry["package"].clone())
12682            })
12683            .collect();
12684        // Pin the count: a corpus that silently shrinks would take its coverage
12685        // with it, and a suite reading N-1 vectors reports the same clean pass
12686        // as one reading N.
12687        assert_eq!(
12688            vectors.len(),
12689            3,
12690            "vendored fed corpus changed size; re-sync from subc-federation"
12691        );
12692
12693        // Pin what makes the corpus DISCRIMINATING, not just present.
12694        //
12695        // The relay test below takes its expected value from the corpus, so the
12696        // corpus supplies the test's power to detect a lossy relay rather than
12697        // its correctness. A relay that dropped unrecognised fields would still
12698        // be caught -- but only by a package carrying fields it does not know.
12699        // Shrink every package to the handful of keys any implementation would
12700        // recognise and the test keeps passing over an input that can no longer
12701        // fail, which is the same clean green as a corpus that shrank away.
12702        //
12703        // So assert the precondition rather than duplicating the packages here:
12704        // at least one vector must carry a field beyond the small common set.
12705        // That is one claim to maintain instead of nine, and it fails loudly if
12706        // a re-sync ever flattens the corpus.
12707        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12708        let richest = vectors
12709            .iter()
12710            .filter_map(|(_, package)| package.as_object())
12711            .map(|object| {
12712                object
12713                    .keys()
12714                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12715                    .count()
12716            })
12717            .max()
12718            .unwrap_or(0);
12719        assert!(
12720            richest >= 2,
12721            "vendored corpus no longer carries a package with unmodelled fields, \
12722             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12723        );
12724
12725        vectors
12726    }
12727
12728    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12729    ///
12730    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12731    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12732    /// three-key object, so a relay that quietly dropped fields it did not
12733    /// recognise would satisfy it. These vectors carry nine keys including ones
12734    /// this crate has no type for, so a typed relay fails here and only here.
12735    #[tokio::test]
12736    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12737        for (corpus_id, package) in fed_admission_facts_vectors() {
12738            let registry = Arc::new(Registry::default());
12739            let forwarding = Arc::new(ForwardingTable::default());
12740            let supervisor = SupervisorHandle::new();
12741            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12742            let handler =
12743                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12744                    .with_supervisor(supervisor)
12745                    .with_admission_facts_config(
12746                        Some("fed".to_string()),
12747                        Some(vec!["target".to_string()]),
12748                    );
12749
12750            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12751            hello_via_sink(
12752                &handler,
12753                &target_ctx,
12754                &mut target_rx,
12755                hello_frame("target", PROTOCOL_VERSION, 1),
12756            )
12757            .await;
12758
12759            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12760            let route_handler = handler.clone();
12761            let expected = package.clone();
12762            let route_task = tokio::spawn(async move {
12763                route_handler
12764                    .handle_control_frame(
12765                        &client_ctx,
12766                        route_open_frame_with_admission_facts(
12767                            20,
12768                            "target",
12769                            unique_project_root("admission-facts"),
12770                            Some(subc_control::ConsumerIdentity {
12771                                module_id: "fed".to_string(),
12772                                launch_nonce: "fed-nonce".to_string(),
12773                            }),
12774                            Some(package),
12775                        ),
12776                    )
12777                    .await
12778                    .unwrap()
12779            });
12780
12781            let bind_frame = target_rx.recv().await.unwrap();
12782            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12783            let ModuleControlRequest::RouteBind {
12784                admission_facts, ..
12785            } = bind
12786            else {
12787                panic!("{corpus_id}: expected route.bind")
12788            };
12789            assert_eq!(
12790                admission_facts,
12791                Some(expected),
12792                "{corpus_id}: relay must not add, drop or reshape any field"
12793            );
12794
12795            handler
12796                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12797                .await
12798                .unwrap();
12799            route_task.await.unwrap();
12800        }
12801    }
12802
12803    #[tokio::test]
12804    async fn route_bind_never_relays_the_openers_build_class() {
12805        let registry = Arc::new(Registry::default());
12806        let forwarding = Arc::new(ForwardingTable::default());
12807        let handler =
12808            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12809        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12810        hello_via_sink(
12811            &handler,
12812            &target_ctx,
12813            &mut target_rx,
12814            hello_frame("target", PROTOCOL_VERSION, 1),
12815        )
12816        .await;
12817
12818        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
12819        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12820        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12821            busy_refusals: None,
12822            target: RouteTarget::ToolProvider {
12823                module_id: "target".to_string(),
12824            },
12825            identity,
12826            consumer_identity: None,
12827            consumer_capabilities: None,
12828            role_versions: None,
12829            admission_facts: None,
12830            scope: None,
12831        })
12832        .unwrap();
12833        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12834
12835        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12836        let route_handler = handler.clone();
12837        let route_task = tokio::spawn(async move {
12838            route_handler
12839                .handle_control_frame(&client_ctx, open)
12840                .await
12841                .unwrap()
12842        });
12843        let bind_frame = target_rx.recv().await.unwrap();
12844        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12845        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12846            panic!("expected route.bind")
12847        };
12848        assert_eq!(
12849            identity.build_class, None,
12850            "route.bind carried the build class"
12851        );
12852        assert!(
12853            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12854            "the route.bind bytes still name build_class"
12855        );
12856        handler
12857            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12858            .await
12859            .unwrap();
12860        route_task.await.unwrap();
12861    }
12862
12863    #[tokio::test]
12864    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12865        let registry = Arc::new(Registry::default());
12866        let forwarding = Arc::new(ForwardingTable::default());
12867        let supervisor = SupervisorHandle::new();
12868        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12869        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12870        let handler =
12871            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12872                .with_supervisor(supervisor)
12873                .with_admission_facts_config(
12874                    Some("fed".to_string()),
12875                    Some(vec!["target".to_string()]),
12876                );
12877
12878        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12879        hello_via_sink(
12880            &handler,
12881            &target_ctx,
12882            &mut target_rx,
12883            hello_frame("target", PROTOCOL_VERSION, 1),
12884        )
12885        .await;
12886        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12887        hello_via_sink(
12888            &handler,
12889            &other_ctx,
12890            &mut other_rx,
12891            hello_frame("other", PROTOCOL_VERSION, 2),
12892        )
12893        .await;
12894
12895        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12896        let expected_facts = facts.clone();
12897        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12898        let route_handler = handler.clone();
12899        let route_task = tokio::spawn(async move {
12900            route_handler
12901                .handle_control_frame(
12902                    &client_ctx,
12903                    route_open_frame_with_admission_facts(
12904                        10,
12905                        "target",
12906                        unique_project_root("admission-facts"),
12907                        Some(subc_control::ConsumerIdentity {
12908                            module_id: "fed".to_string(),
12909                            launch_nonce: "fed-nonce".to_string(),
12910                        }),
12911                        Some(facts.clone()),
12912                    ),
12913                )
12914                .await
12915                .unwrap()
12916        });
12917        let bind_frame = target_rx.recv().await.unwrap();
12918        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12919        let ModuleControlRequest::RouteBind {
12920            admission_facts, ..
12921        } = bind
12922        else {
12923            panic!("expected route.bind")
12924        };
12925        assert_eq!(admission_facts, Some(expected_facts));
12926        handler
12927            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12928            .await
12929            .unwrap();
12930        assert!(route_task.await.unwrap().is_empty());
12931        assert!(matches!(
12932            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12933                .unwrap(),
12934            ClientControlResponse::RouteOpen { .. }
12935        ));
12936
12937        let direct = handler
12938            .handle_control_frame(
12939                &route_ctx(ConnectionId::new(73)).0,
12940                route_open_frame_with_admission_facts(
12941                    11,
12942                    "target",
12943                    unique_project_root("admission-facts"),
12944                    None,
12945                    Some(json!({"x": 1})),
12946                ),
12947            )
12948            .await
12949            .unwrap();
12950        assert_eq!(
12951            parse_error(&direct[0])["code"],
12952            "admission_facts_not_permitted"
12953        );
12954
12955        let different_reserved = handler
12956            .handle_control_frame(
12957                &route_ctx(ConnectionId::new(77)).0,
12958                route_open_frame_with_admission_facts(
12959                    15,
12960                    "target",
12961                    unique_project_root("admission-facts"),
12962                    Some(subc_control::ConsumerIdentity {
12963                        module_id: "other".to_string(),
12964                        launch_nonce: "other-nonce".to_string(),
12965                    }),
12966                    Some(json!({"x": 1})),
12967                ),
12968            )
12969            .await
12970            .unwrap();
12971        assert_eq!(
12972            parse_error(&different_reserved[0])["code"],
12973            "admission_facts_not_permitted"
12974        );
12975
12976        let other_target = handler
12977            .handle_control_frame(
12978                &route_ctx(ConnectionId::new(74)).0,
12979                route_open_frame_with_admission_facts(
12980                    12,
12981                    "other",
12982                    unique_project_root("admission-facts"),
12983                    Some(subc_control::ConsumerIdentity {
12984                        module_id: "fed".to_string(),
12985                        launch_nonce: "fed-nonce".to_string(),
12986                    }),
12987                    Some(json!({"x": 1})),
12988                ),
12989            )
12990            .await
12991            .unwrap();
12992        assert_eq!(
12993            parse_error(&other_target[0])["code"],
12994            "admission_facts_target_not_allowed"
12995        );
12996
12997        let nonexistent = handler
12998            .handle_control_frame(
12999                &route_ctx(ConnectionId::new(75)).0,
13000                route_open_frame_with_admission_facts(
13001                    13,
13002                    "missing",
13003                    unique_project_root("admission-facts"),
13004                    None,
13005                    Some(json!({"x": 1})),
13006                ),
13007            )
13008            .await
13009            .unwrap();
13010        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13011
13012        let described = handler
13013            .handle_control_frame(
13014                &route_ctx(ConnectionId::new(76)).0,
13015                Frame::build(
13016                    FrameType::Request,
13017                    control_flags(),
13018                    0,
13019                    0,
13020                    14,
13021                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13022                )
13023                .unwrap(),
13024            )
13025            .await
13026            .unwrap();
13027        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13028            serde_json::from_slice(&described[0].body).unwrap()
13029        else {
13030            panic!("expected server.describe response")
13031        };
13032        assert!(capabilities
13033            .iter()
13034            .any(|cap| cap == "admission_facts_relay_v1"));
13035    }
13036
13037    #[tokio::test]
13038    async fn admission_facts_without_configured_carrier_are_rejected() {
13039        let registry = Arc::new(Registry::default());
13040        let forwarding = Arc::new(ForwardingTable::default());
13041        let handler = ControlHandler::with_forwarding(registry, forwarding);
13042        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13043        hello_via_sink(
13044            &handler,
13045            &target_ctx,
13046            &mut target_rx,
13047            hello_frame("target", PROTOCOL_VERSION, 1),
13048        )
13049        .await;
13050
13051        let responses = handler
13052            .handle_control_frame(
13053                &route_ctx(ConnectionId::new(79)).0,
13054                route_open_frame_with_admission_facts(
13055                    16,
13056                    "target",
13057                    unique_project_root("admission-facts"),
13058                    None,
13059                    Some(json!({"x": 1})),
13060                ),
13061            )
13062            .await
13063            .unwrap();
13064        assert_eq!(
13065            parse_error(&responses[0])["code"],
13066            "admission_facts_not_permitted"
13067        );
13068    }
13069
13070    #[tokio::test]
13071    async fn route_open_relays_consumer_capabilities_verbatim() {
13072        let registry = Arc::new(Registry::default());
13073        let forwarding = Arc::new(ForwardingTable::default());
13074        let handler =
13075            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13076        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13077        hello_via_sink(
13078            &handler,
13079            &module_ctx,
13080            &mut module_rx,
13081            hello_frame("aft", PROTOCOL_VERSION, 7),
13082        )
13083        .await;
13084
13085        let expected = vec!["elicitation".to_string(), "roots".to_string()];
13086        let expected_for_request = expected.clone();
13087        let project_root = unique_project_root("consumer-capabilities-present");
13088        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13089        let route_handler = handler.clone();
13090        let route_task = tokio::spawn(async move {
13091            route_handler
13092                .handle_control_frame(
13093                    &client_ctx,
13094                    route_open_frame_with_consumer_capabilities(
13095                        401,
13096                        "aft",
13097                        project_root,
13098                        Some(expected_for_request),
13099                    ),
13100                )
13101                .await
13102                .unwrap()
13103        });
13104        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13105            .await
13106            .unwrap()
13107            .unwrap();
13108        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13109        let ModuleControlRequest::RouteBind {
13110            consumer_capabilities,
13111            ..
13112        } = bind
13113        else {
13114            panic!("expected route.bind request, got {bind:?}");
13115        };
13116        assert_eq!(consumer_capabilities, Some(expected.clone()));
13117
13118        handler
13119            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13120            .await
13121            .unwrap();
13122        let route_response = route_task.await.unwrap();
13123        assert!(route_response.is_empty());
13124        let published = client_rx.recv().await.unwrap();
13125        assert!(matches!(
13126            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13127            ClientControlResponse::RouteOpen { .. }
13128        ));
13129    }
13130
13131    #[tokio::test]
13132    async fn route_open_without_consumer_capabilities_relays_none() {
13133        let registry = Arc::new(Registry::default());
13134        let forwarding = Arc::new(ForwardingTable::default());
13135        let handler =
13136            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13137        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13138        hello_via_sink(
13139            &handler,
13140            &module_ctx,
13141            &mut module_rx,
13142            hello_frame("aft", PROTOCOL_VERSION, 7),
13143        )
13144        .await;
13145
13146        let project_root = unique_project_root("consumer-capabilities-absent");
13147        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13148        let route_handler = handler.clone();
13149        let route_task = tokio::spawn(async move {
13150            route_handler
13151                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13152                .await
13153                .unwrap()
13154        });
13155        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13156            .await
13157            .unwrap()
13158            .unwrap();
13159        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13160        let ModuleControlRequest::RouteBind {
13161            consumer_capabilities,
13162            ..
13163        } = bind
13164        else {
13165            panic!("expected route.bind request, got {bind:?}");
13166        };
13167        assert_eq!(consumer_capabilities, None);
13168
13169        handler
13170            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13171            .await
13172            .unwrap();
13173        let route_response = route_task.await.unwrap();
13174        assert!(route_response.is_empty());
13175        let published = client_rx.recv().await.unwrap();
13176        assert!(matches!(
13177            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13178            ClientControlResponse::RouteOpen { .. }
13179        ));
13180    }
13181
13182    /// Opens a route to a freshly registered `aft` with `sent` as the
13183    /// route.open's role_versions, acks the bind, and returns the role_versions
13184    /// the module's bind carried.
13185    async fn bind_role_versions_for(
13186        sent: Option<BTreeMap<String, String>>,
13187        connection: u64,
13188    ) -> Option<BTreeMap<String, String>> {
13189        let registry = Arc::new(Registry::default());
13190        let forwarding = Arc::new(ForwardingTable::default());
13191        let handler =
13192            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13193        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13194        hello_via_sink(
13195            &handler,
13196            &module_ctx,
13197            &mut module_rx,
13198            hello_frame("aft", PROTOCOL_VERSION, 7),
13199        )
13200        .await;
13201        let project_root = unique_project_root("role-versions");
13202        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13203        let route_handler = handler.clone();
13204        let route_task = tokio::spawn(async move {
13205            route_handler
13206                .handle_control_frame(
13207                    &client_ctx,
13208                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
13209                )
13210                .await
13211                .unwrap()
13212        });
13213        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13214            .await
13215            .expect("a well-formed route.open reaches the module as a bind")
13216            .unwrap();
13217        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13218        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13219            panic!("expected route.bind request, got {bind:?}");
13220        };
13221        handler
13222            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13223            .await
13224            .unwrap();
13225        assert!(route_task.await.unwrap().is_empty());
13226        let published = client_rx.recv().await.unwrap();
13227        assert!(matches!(
13228            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13229            ClientControlResponse::RouteOpen { .. }
13230        ));
13231        role_versions
13232    }
13233
13234    #[tokio::test]
13235    async fn route_open_relays_role_versions_verbatim() {
13236        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13237        assert_eq!(
13238            bind_role_versions_for(Some(sent.clone()), 141).await,
13239            Some(sent)
13240        );
13241    }
13242
13243    /// An empty map declares nothing, so the provider sees no field rather
13244    /// than an empty object it would have to treat as a second "none".
13245    #[tokio::test]
13246    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13247        assert_eq!(bind_role_versions_for(None, 143).await, None);
13248        assert_eq!(
13249            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13250            None
13251        );
13252    }
13253
13254    #[tokio::test]
13255    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13256        let registry = Arc::new(Registry::default());
13257        let forwarding = Arc::new(ForwardingTable::default());
13258        let handler =
13259            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13260        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13261        hello_via_sink(
13262            &handler,
13263            &module_ctx,
13264            &mut module_rx,
13265            hello_frame("aft", PROTOCOL_VERSION, 7),
13266        )
13267        .await;
13268
13269        let nine: BTreeMap<String, String> = (0..9)
13270            .map(|index| (format!("role-{index}"), "v1".to_string()))
13271            .collect();
13272        for (label, malformed) in [
13273            (
13274                "invalid role name",
13275                role_versions(&[("Tool_Provider", "v1")]),
13276            ),
13277            ("invalid version", role_versions(&[("tool-provider", "v0")])),
13278            ("nine entries", nine),
13279        ] {
13280            // A refused open answers at once; one that reached the module
13281            // would wait for its bind ack and trip this timeout.
13282            let responses = tokio::time::timeout(
13283                Duration::from_secs(1),
13284                handler.handle_control_frame(
13285                    &route_ctx(ConnectionId::new(148)).0,
13286                    route_open_frame_with_role_versions(
13287                        404,
13288                        "aft",
13289                        unique_project_root("role-versions-malformed"),
13290                        Some(malformed),
13291                    ),
13292                ),
13293            )
13294            .await
13295            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13296            .unwrap();
13297            assert_eq!(responses.len(), 1, "{label}");
13298            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13299            let error = parse_error(&responses[0]);
13300            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13301            assert_eq!(
13302                error["detail"]["field"], "role_versions",
13303                "{label}: {error}"
13304            );
13305            assert!(
13306                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13307                "{label}: a malformed declaration is terminal"
13308            );
13309            assert!(
13310                module_rx.try_recv().is_err(),
13311                "{label}: the module must never see a bind"
13312            );
13313        }
13314    }
13315
13316    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
13317    /// consumer can tell this daemon forwards the field from one that would
13318    /// drop it.
13319    #[tokio::test]
13320    async fn route_role_versions_capability_is_advertised() {
13321        let handler = ControlHandler::new(Arc::new(Registry::default()));
13322        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13323        let ack = hello_via_sink(
13324            &handler,
13325            &ctx,
13326            &mut rx,
13327            hello_frame("m", PROTOCOL_VERSION, 1),
13328        )
13329        .await;
13330        let ack = parse_ack(&ack);
13331        assert!(
13332            ack.subc_capabilities
13333                .iter()
13334                .any(|c| c == "route-role-versions/v1"),
13335            "{:?}",
13336            ack.subc_capabilities
13337        );
13338
13339        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13340        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13341        let reply = handler
13342            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13343            .await
13344            .unwrap()
13345            .pop()
13346            .unwrap();
13347        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13348            serde_json::from_slice(&reply.body).unwrap()
13349        else {
13350            panic!("not a server.describe reply");
13351        };
13352        assert!(
13353            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13354            "{capabilities:?}"
13355        );
13356    }
13357
13358    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
13359    /// catalog entry without `events` (the module declares none) from a daemon
13360    /// that never reports declared events.
13361    #[tokio::test]
13362    async fn catalog_events_capability_is_advertised() {
13363        let handler = ControlHandler::new(Arc::new(Registry::default()));
13364        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13365        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13366        let reply = handler
13367            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13368            .await
13369            .unwrap()
13370            .pop()
13371            .unwrap();
13372        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13373            serde_json::from_slice(&reply.body).unwrap()
13374        else {
13375            panic!("not a server.describe reply");
13376        };
13377        assert!(
13378            capabilities
13379                .iter()
13380                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13381            "{capabilities:?}"
13382        );
13383    }
13384
13385    #[tokio::test]
13386    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13387        let handler = ControlHandler::new(Arc::new(Registry::default()));
13388        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13389        let ack = hello_via_sink(
13390            &handler,
13391            &module,
13392            &mut module_rx,
13393            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13394        )
13395        .await;
13396        let ack = parse_ack(&ack);
13397        assert!(
13398            ack.subc_capabilities
13399                .iter()
13400                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13401            "{:?}",
13402            ack.subc_capabilities
13403        );
13404        assert!(ack
13405            .subc_capabilities
13406            .iter()
13407            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13408
13409        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13410        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13411        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13412        let reply = handler
13413            .handle_control_frame(&client, frame)
13414            .await
13415            .unwrap()
13416            .pop()
13417            .unwrap();
13418        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13419            serde_json::from_slice(&reply.body).unwrap()
13420        else {
13421            panic!("not a server.describe reply");
13422        };
13423        assert!(
13424            capabilities
13425                .iter()
13426                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13427            "{capabilities:?}"
13428        );
13429        assert!(capabilities
13430            .iter()
13431            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13432    }
13433
13434    #[tokio::test]
13435    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13436        let handler = ControlHandler::new(Arc::new(Registry::default()));
13437        let (module, mut rx) = route_ctx(ConnectionId::new(156));
13438        let ack = hello_via_sink(
13439            &handler,
13440            &module,
13441            &mut rx,
13442            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13443        )
13444        .await;
13445        assert!(parse_ack(&ack)
13446            .subc_capabilities
13447            .iter()
13448            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13449        assert!(parse_ack(&ack)
13450            .subc_capabilities
13451            .iter()
13452            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13453        let (client, _rx) = route_ctx(ConnectionId::new(157));
13454        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13455        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13456        let reply = handler
13457            .handle_control_frame(&client, frame)
13458            .await
13459            .unwrap()
13460            .pop()
13461            .unwrap();
13462        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13463            serde_json::from_slice(&reply.body).unwrap()
13464        else {
13465            panic!("server.describe reply")
13466        };
13467        assert!(capabilities
13468            .iter()
13469            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13470        assert!(capabilities
13471            .iter()
13472            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13473    }
13474
13475    #[tokio::test]
13476    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13477        let handler = ControlHandler::new(Arc::new(Registry::default()));
13478        let (module, mut rx) = route_ctx(ConnectionId::new(158));
13479        let ack = hello_via_sink(
13480            &handler,
13481            &module,
13482            &mut rx,
13483            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13484        )
13485        .await;
13486        assert!(parse_ack(&ack)
13487            .subc_capabilities
13488            .iter()
13489            .any(|cap| cap == "scope-workdir/v1"));
13490        let (client, _rx) = route_ctx(ConnectionId::new(159));
13491        let frame = Frame::build(
13492            FrameType::Request,
13493            control_flags(),
13494            0,
13495            0,
13496            5,
13497            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13498        )
13499        .unwrap();
13500        let reply = handler
13501            .handle_control_frame(&client, frame)
13502            .await
13503            .unwrap()
13504            .remove(0);
13505        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13506            serde_json::from_slice(&reply.body).unwrap()
13507        else {
13508            panic!("server.describe reply")
13509        };
13510        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13511    }
13512
13513    #[tokio::test]
13514    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13515        let handler = ControlHandler::new(Arc::new(Registry::default()));
13516        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13517        let ack = hello_via_sink(
13518            &handler,
13519            &module,
13520            &mut rx,
13521            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13522        )
13523        .await;
13524        assert!(parse_ack(&ack)
13525            .subc_capabilities
13526            .iter()
13527            .any(|cap| cap == "worker-runs/v1"));
13528        let (client, _rx) = route_ctx(ConnectionId::new(161));
13529        let frame = Frame::build(
13530            FrameType::Request,
13531            control_flags(),
13532            0,
13533            0,
13534            5,
13535            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13536        )
13537        .unwrap();
13538        let reply = handler
13539            .handle_control_frame(&client, frame)
13540            .await
13541            .unwrap()
13542            .remove(0);
13543        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13544            serde_json::from_slice(&reply.body).unwrap()
13545        else {
13546            panic!("server.describe reply")
13547        };
13548        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13549    }
13550
13551    #[tokio::test]
13552    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13553        let handler = ControlHandler::new(Arc::new(Registry::default()));
13554        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13555        let ack = hello_via_sink(
13556            &handler,
13557            &module,
13558            &mut rx,
13559            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13560        )
13561        .await;
13562        assert!(parse_ack(&ack)
13563            .subc_capabilities
13564            .iter()
13565            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13566        let (client, _rx) = route_ctx(ConnectionId::new(155));
13567        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13568        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13569        let reply = handler
13570            .handle_control_frame(&client, frame)
13571            .await
13572            .unwrap()
13573            .pop()
13574            .unwrap();
13575        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13576            serde_json::from_slice(&reply.body).unwrap()
13577        else {
13578            panic!("not a server.describe reply")
13579        };
13580        assert!(capabilities
13581            .iter()
13582            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13583    }
13584
13585    #[tokio::test]
13586    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13587        let registry = Arc::new(Registry::default());
13588        let forwarding = Arc::new(ForwardingTable::default());
13589        let handler =
13590            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13591                .with_health_probe_timeout(Duration::from_secs(5));
13592        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13593        hello_via_sink(
13594            &handler,
13595            &module_ctx,
13596            &mut module_rx,
13597            non_routable_hello_frame_with_control_ops(
13598                "mcp",
13599                300,
13600                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13601            ),
13602        )
13603        .await;
13604        assert!(registry
13605            .get_module("mcp")
13606            .unwrap()
13607            .unwrap()
13608            .manifest
13609            .provides
13610            .is_empty());
13611
13612        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13613        let route_response = handler
13614            .handle_control_frame(
13615                &route_client_ctx,
13616                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13617            )
13618            .await
13619            .unwrap();
13620        assert_eq!(route_response[0].header.ty, FrameType::Error);
13621        assert_eq!(
13622            parse_error(&route_response[0])["code"],
13623            "target_unavailable"
13624        );
13625        assert!(parse_error(&route_response[0])["message"]
13626            .as_str()
13627            .unwrap()
13628            .contains("does not provide the requested target"));
13629        assert!(module_rx.try_recv().is_err());
13630
13631        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13632        let health_handler = handler.clone();
13633        let health_task = tokio::spawn(async move {
13634            health_handler
13635                .handle_control_frame(
13636                    &health_client_ctx,
13637                    supervisor_health_probe_frame(302, "mcp"),
13638                )
13639                .await
13640                .unwrap()
13641        });
13642        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13643            .await
13644            .unwrap()
13645            .unwrap();
13646        assert_eq!(
13647            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13648            ModuleControlRequest::HealthCheck {}
13649        );
13650        handler
13651            .handle_control_frame(
13652                &module_ctx,
13653                health_response(health_frame.header.corr, HealthStatus::Ok),
13654            )
13655            .await
13656            .unwrap();
13657        let health_response = health_task.await.unwrap();
13658        assert_eq!(health_response[0].header.ty, FrameType::Response);
13659        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13660            ClientControlResponse::SupervisorHealthProbe {
13661                module_id, status, ..
13662            } => {
13663                assert_eq!(module_id, "mcp");
13664                assert_eq!(status, HealthStatus::Ok);
13665            }
13666            other => panic!("unexpected health response: {other:?}"),
13667        }
13668
13669        // Exercise the forwarding cleanup path directly while leaving the registry
13670        // advertisement in place. If cleanup leaves a stale control sink behind,
13671        // the next probe will enqueue onto it and wait for the long probe timeout
13672        // instead of returning an immediate no-connection error.
13673        forwarding
13674            .cleanup_connection(module_ctx.connection_id)
13675            .unwrap();
13676        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13677        let cleanup_response = tokio::time::timeout(
13678            Duration::from_millis(200),
13679            handler.handle_control_frame(
13680                &cleanup_probe_ctx,
13681                supervisor_health_probe_frame(303, "mcp"),
13682            ),
13683        )
13684        .await
13685        .expect("probe should fail immediately when the control lane is gone")
13686        .unwrap();
13687        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13688        assert_eq!(
13689            parse_error(&cleanup_response[0])["code"],
13690            "target_unavailable"
13691        );
13692        assert!(parse_error(&cleanup_response[0])["message"]
13693            .as_str()
13694            .unwrap()
13695            .contains("no module connection"));
13696
13697        handler
13698            .cleanup_connection(module_ctx.connection_id)
13699            .unwrap();
13700    }
13701
13702    #[tokio::test]
13703    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13704        let registry = Arc::new(Registry::default());
13705        let supervisor_handle = SupervisorHandle::new();
13706        let supervisor =
13707            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13708                .with_handle(supervisor_handle.clone())
13709                .with_connection_file_path(
13710                    std::env::temp_dir()
13711                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13712                );
13713        let module = supervisor
13714            .supervise_configured(
13715                ModuleSpec {
13716                    carry_containment: false,
13717                    module_id: "warming".to_string(),
13718                    program: fake_aft_stub_path(),
13719                    args: Vec::new(),
13720                    env: Vec::new(),
13721                    reserved: false,
13722                    reserved_prefixes: Vec::new(),
13723                    protocol: ModuleProtocol::Subc,
13724                    overlap: Default::default(),
13725                },
13726                true,
13727            )
13728            .unwrap();
13729        assert_eq!(module.state().unwrap(), ModuleState::Running);
13730
13731        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13732        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13733        let response = handler
13734            .handle_control_frame(
13735                &ctx,
13736                route_open_frame(304, "warming", unique_project_root("warming")),
13737            )
13738            .await
13739            .unwrap();
13740        module.stop().await.unwrap();
13741
13742        assert_eq!(response[0].header.ty, FrameType::Error);
13743        let error = parse_error(&response[0]);
13744        assert_eq!(error["code"], "module_warming");
13745        assert!(error["message"]
13746            .as_str()
13747            .unwrap()
13748            .contains("state=running, enabled=true, live=false"));
13749    }
13750
13751    #[test]
13752    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13753        let handler = ControlHandler::new(Arc::new(Registry::default()));
13754        let capture = EventCapture::default();
13755        let _subscriber =
13756            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13757        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13758        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13759        let pending = (0..limit).collect::<Vec<_>>();
13760        let response = handler
13761            .route_open_capacity_refusal(
13762                &ctx,
13763                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13764                "busy",
13765                pending.len(),
13766                limit,
13767            )
13768            .unwrap();
13769        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13770        let event = capture
13771            .events()
13772            .into_iter()
13773            .find(|event| {
13774                event.target == "control"
13775                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13776            })
13777            .expect("connection admission refusal event");
13778        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13779        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13780        // Refused before `handle_route_open` parses the request, so the caller
13781        // fields are read from the frame itself.
13782        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13783        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13784    }
13785
13786    #[test]
13787    fn route_open_target_cap_logs_admission_reason_and_capacity() {
13788        let handler = ControlHandler::new(Arc::new(Registry::default()));
13789        let capture = EventCapture::default();
13790        let _subscriber =
13791            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13792        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13793        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13794        let guards = (0..limit)
13795            .map(|_| {
13796                handler
13797                    .route_bind_concurrency
13798                    .try_admit("busy", limit)
13799                    .unwrap()
13800            })
13801            .collect::<Vec<_>>();
13802        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13803            Err(in_flight) => in_flight,
13804            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13805        };
13806        let response = handler
13807            .route_open_target_capacity_refusal(
13808                &ctx,
13809                &RouteOpenCaller::unknown(),
13810                &route_open_frame(397, "busy", unique_project_root("target-cap")),
13811                "busy",
13812                in_flight,
13813            )
13814            .unwrap();
13815        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13816        let event = capture
13817            .events()
13818            .into_iter()
13819            .find(|event| {
13820                event.target == "control"
13821                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13822            })
13823            .expect("target admission refusal event");
13824        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13825        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13826        drop(guards);
13827    }
13828
13829    #[cfg(unix)]
13830    #[tokio::test(start_paused = true)]
13831    async fn supervisor_list_shows_a_failed_modules_next_retry() {
13832        let registry = Arc::new(Registry::default());
13833        let handle = SupervisorHandle::new();
13834        let supervisor =
13835            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13836                .with_handle(handle.clone());
13837        let module = supervisor
13838            .spawn(crate::ModuleSpec {
13839                carry_containment: false,
13840                module_id: "retry-list".into(),
13841                program: PathBuf::from("/bin/sh"),
13842                args: vec!["-c".into(), "exit 17".into()],
13843                env: vec![],
13844                reserved: false,
13845                reserved_prefixes: vec![],
13846                protocol: ModuleProtocol::None,
13847                overlap: Default::default(),
13848            })
13849            .unwrap();
13850        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13851        while module.state().unwrap() != crate::ModuleState::Failed {
13852            assert!(
13853                std::time::Instant::now() < deadline,
13854                "waiting for budget failure"
13855            );
13856            tokio::task::yield_now().await;
13857        }
13858        let handler = ControlHandler::new(registry).with_supervisor(handle);
13859        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13860        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13861        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13862        assert_eq!(body["modules"][0]["state"], "failed");
13863        let retry = body["modules"][0]["next_retry_at_ms"]
13864            .as_u64()
13865            .expect("scheduled retry on the wire");
13866        let now = std::time::SystemTime::now()
13867            .duration_since(std::time::UNIX_EPOCH)
13868            .unwrap()
13869            .as_millis() as u64;
13870        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13871        module.set_enabled(false).await.unwrap();
13872        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13873        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13874        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13875        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13876    }
13877
13878    #[tokio::test]
13879    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
13880        let _guard = crate::subtree_resources::fault_read_guard().await;
13881        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
13882        let supervisor =
13883            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13884        let module = supervisor
13885            .spawn(crate::ModuleSpec {
13886                carry_containment: false,
13887                module_id: "pid-read-budget".into(),
13888                program: std::env::current_exe().unwrap(),
13889                args: vec![
13890                    "--exact".into(),
13891                    "control::tests::pid_resource_read_fixture".into(),
13892                ],
13893                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13894                    .into_iter()
13895                    .map(|key| (key.into(), dir.display().to_string()))
13896                    .chain(std::iter::once((
13897                        "SUBC_SUBTREE_PID_FIXTURE".into(),
13898                        "1".into(),
13899                    )))
13900                    .collect(),
13901                reserved: false,
13902                reserved_prefixes: Vec::new(),
13903                protocol: ModuleProtocol::None,
13904                overlap: Default::default(),
13905            })
13906            .unwrap();
13907        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
13908        while !matches!(
13909            module.child_resource_usage(),
13910            subc_control::ChildResourceUsage::Measured(_)
13911        ) {
13912            assert!(
13913                tokio::time::Instant::now() < ready_deadline,
13914                "fixture process was not measurable"
13915            );
13916            tokio::time::sleep(Duration::from_millis(10)).await;
13917        }
13918        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
13919        let reading = crate::subtree_resources::read_process(module.clone()).await;
13920        module.stop().await.unwrap();
13921        assert!(
13922            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
13923            "{reading:?}"
13924        );
13925    }
13926
13927    #[test]
13928    fn pid_resource_read_fixture() {
13929        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
13930            loop {
13931                std::thread::park_timeout(Duration::from_secs(600));
13932            }
13933        }
13934    }
13935
13936    #[tokio::test]
13937    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
13938        let _guard = crate::subtree_resources::fault_read_guard().await;
13939        let registry = Arc::new(Registry::default());
13940        let handle = SupervisorHandle::new();
13941        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13942            .with_handle(handle.clone());
13943        let module = supervisor
13944            .supervise_configured(
13945                crate::ModuleSpec {
13946                    carry_containment: false,
13947                    module_id: "memory-read".into(),
13948                    program: PathBuf::from("disabled-module"),
13949                    args: Vec::new(),
13950                    env: Vec::new(),
13951                    reserved: false,
13952                    reserved_prefixes: Vec::new(),
13953                    protocol: ModuleProtocol::Subc,
13954                    overlap: Default::default(),
13955                },
13956                false,
13957            )
13958            .unwrap();
13959        let (target, release, began) = crate::subtree_resources::held();
13960        module.set_subtree_target_for_test(target);
13961        let handler = ControlHandler::new(registry).with_supervisor(handle);
13962        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
13963        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13964        let replies = handler
13965            .handle_client_control_request(
13966                &ctx,
13967                frame,
13968                ClientControlRequest::SupervisorList {},
13969                &mut None,
13970            )
13971            .await
13972            .unwrap();
13973        assert!(
13974            replies.is_empty(),
13975            "list reply must be detached from the connection reader"
13976        );
13977        began.await.unwrap();
13978        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13979        let replies = handler
13980            .handle_client_control_request(
13981                &ctx,
13982                frame,
13983                ClientControlRequest::SupervisorSpawnSnapshot {},
13984                &mut None,
13985            )
13986            .await
13987            .unwrap();
13988        assert_eq!(
13989            replies.len(),
13990            1,
13991            "another request must complete while the subtree read stalls"
13992        );
13993        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
13994            .await
13995            .unwrap()
13996            .unwrap();
13997        release.send(()).unwrap();
13998        drop(release);
13999        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14000        assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14001        assert_eq!(body["modules"][0]["subtree"]["reason"], "timeout");
14002        assert_eq!(
14003            body["modules"][0]["subtree"]["process_count"], 1,
14004            "a count obtained before the stalled read must survive timeout"
14005        );
14006        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14007        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14008        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14009        assert_eq!(
14010            body["modules"][0]["subtree"]["source"], "process_sum",
14011            "the next request must read afresh"
14012        );
14013    }
14014
14015    #[tokio::test]
14016    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14017        let registry = Arc::new(Registry::default());
14018        let handle = SupervisorHandle::new();
14019        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14020            .with_handle(handle.clone());
14021        supervisor
14022            .supervise_configured(
14023                crate::ModuleSpec {
14024                    carry_containment: false,
14025                    module_id: "drift".into(),
14026                    program: PathBuf::from("disabled-module"),
14027                    args: Vec::new(),
14028                    env: Vec::new(),
14029                    reserved: false,
14030                    reserved_prefixes: Vec::new(),
14031                    protocol: ModuleProtocol::Subc,
14032                    overlap: Default::default(),
14033                },
14034                false,
14035            )
14036            .unwrap();
14037        let handler = ControlHandler::new(registry).with_supervisor(handle);
14038        for configured in [None, Some(false), Some(true)] {
14039            handler.capability_evaluator.configure(
14040                configured.map(|enabled| ("drift".to_string(), enabled)),
14041                BTreeMap::new(),
14042            );
14043            let frame =
14044                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14045            let replies = handler
14046                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14047                .await
14048                .unwrap();
14049            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14050            assert!(body.get("configured_enabled").is_none());
14051            assert_eq!(body["op"], "supervisor.ack");
14052            let frame =
14053                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14054            let replies = handler.handle_supervisor_list(frame).await.unwrap();
14055            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14056            assert_eq!(
14057                body["modules"][0].get("configured_enabled"),
14058                configured.map(Value::Bool).as_ref()
14059            );
14060            assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14061            assert_eq!(body["modules"][0]["subtree"]["reason"], "not_running");
14062        }
14063    }
14064
14065    #[tokio::test(flavor = "current_thread")]
14066    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14067        let handler = ControlHandler::new(Arc::new(Registry::default()));
14068        let capture = EventCapture::default();
14069        let _subscriber =
14070            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14071        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14072        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14073
14074        let _ = handler
14075            .handle_client_control_request(
14076                &ctx,
14077                frame,
14078                ClientControlRequest::SupervisorSetEnabled {
14079                    module_id: "broca".to_string(),
14080                    enabled: false,
14081                },
14082                &mut None,
14083            )
14084            .await
14085            .unwrap();
14086
14087        let events = capture
14088            .events()
14089            .into_iter()
14090            .filter(|event| {
14091                event.target == "control"
14092                    && event.fields.get("message").map(String::as_str)
14093                        == Some("supervisor request received")
14094            })
14095            .collect::<Vec<_>>();
14096        assert_eq!(events.len(), 1, "one supervisor request log line");
14097        let event = &events[0];
14098        assert_eq!(event.level, tracing::Level::INFO);
14099        assert_eq!(
14100            event.fields.get("op"),
14101            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14102        );
14103        assert_eq!(
14104            event.fields.get("module_id"),
14105            Some(&"\"broca\"".to_string())
14106        );
14107        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14108        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14109        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14110    }
14111
14112    #[tokio::test(flavor = "current_thread")]
14113    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14114        let registry = Arc::new(Registry::default());
14115        let module_connection = ConnectionId::new(303);
14116        registry
14117            .register_with_control_ops(
14118                manifest("aft", PROTOCOL_VERSION),
14119                PROTOCOL_VERSION,
14120                module_connection,
14121                module_baseline_control_ops(),
14122            )
14123            .unwrap();
14124        let handler = ControlHandler::new(registry);
14125        let capture = EventCapture::default();
14126        let _subscriber =
14127            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14128        let (ctx, _rx) = route_ctx(module_connection);
14129        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14130
14131        let _ = handler
14132            .handle_client_control_request(
14133                &ctx,
14134                frame,
14135                ClientControlRequest::SupervisorSetEnabled {
14136                    module_id: "broca".to_string(),
14137                    enabled: false,
14138                },
14139                &mut None,
14140            )
14141            .await
14142            .unwrap();
14143
14144        let callers = capture
14145            .events()
14146            .into_iter()
14147            .filter(|event| {
14148                event.fields.get("message").map(String::as_str)
14149                    == Some("supervisor request received")
14150            })
14151            .map(|event| event.fields.get("caller").cloned())
14152            .collect::<Vec<_>>();
14153        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14154    }
14155
14156    #[tokio::test(flavor = "current_thread")]
14157    async fn supervisor_list_does_not_log_request_received() {
14158        let handler = ControlHandler::new(Arc::new(Registry::default()));
14159        let capture = EventCapture::default();
14160        let _subscriber =
14161            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14162        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14163        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14164
14165        handler
14166            .handle_client_control_request(
14167                &ctx,
14168                frame,
14169                ClientControlRequest::SupervisorList {},
14170                &mut None,
14171            )
14172            .await
14173            .unwrap();
14174
14175        assert!(capture.events().into_iter().all(|event| {
14176            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14177        }));
14178    }
14179
14180    #[tokio::test(flavor = "current_thread")]
14181    async fn supervisor_rescan_preview_does_not_log_request_received() {
14182        let handler = ControlHandler::new(Arc::new(Registry::default()));
14183        let capture = EventCapture::default();
14184        let _subscriber =
14185            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14186        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14187        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14188
14189        handler
14190            .handle_client_control_request(
14191                &ctx,
14192                frame,
14193                ClientControlRequest::SupervisorRescan { preview: true },
14194                &mut None,
14195            )
14196            .await
14197            .unwrap();
14198
14199        assert!(capture.events().into_iter().all(|event| {
14200            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14201        }));
14202    }
14203
14204    struct ApproveOperator;
14205    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14206        fn prompt(
14207            &self,
14208            _: &str,
14209            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14210        ) -> crate::operator_confirm::ProviderResult {
14211            crate::operator_confirm::ProviderResult::Approved
14212        }
14213    }
14214
14215    #[tokio::test]
14216    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14217        assert_eq!(
14218            ControlHandler::default().privileged_operator_prompt,
14219            PrivilegedOperatorPrompt::Log
14220        );
14221        for (initial, changed, running) in [
14222            ("log", "enforce", PrivilegedOperatorPrompt::Log),
14223            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14224        ] {
14225            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14226            let path = dir.join("subc.jsonc");
14227            std::fs::write(
14228                &path,
14229                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14230            )
14231            .unwrap();
14232            let config = crate::daemon_config::load(&path).unwrap().unwrap();
14233            let registry = Arc::new(Registry::default());
14234            let handle = SupervisorHandle::new();
14235            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14236                .with_handle(handle.clone());
14237            let handler = ControlHandler::new(registry)
14238                .with_supervisor(handle)
14239                .with_privileged_operator_prompt(config.privileged_operator_prompt)
14240                .with_supervisor_rescan(supervisor, &path, config.port);
14241            handler.forwarding.operator_confirms().configure(
14242                Arc::new(ApproveOperator),
14243                crate::operator_confirm::OperatorLimits::default(),
14244            );
14245            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14246
14247            for (disk_mode, needs_restart) in [
14248                (Some(initial), false),
14249                (Some(changed), true),
14250                (Some(changed), true),
14251                (None, running == PrivilegedOperatorPrompt::Enforce),
14252                (Some(initial), false),
14253            ] {
14254                let mut doc = json!({"version":1});
14255                if let Some(mode) = disk_mode {
14256                    doc["privileged_operator_prompt"] = json!(mode);
14257                }
14258                std::fs::write(&path, doc.to_string()).unwrap();
14259                for preview in [true, false] {
14260                    let request = ClientControlRequest::SupervisorRescan { preview };
14261                    let frame = Frame::build(
14262                        FrameType::Request,
14263                        control_flags(),
14264                        0,
14265                        0,
14266                        1,
14267                        serde_json::to_vec(&request).unwrap(),
14268                    )
14269                    .unwrap();
14270                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14271                    assert_eq!(frames.len(), 1);
14272                    let ClientControlResponse::SupervisorRescan { result } =
14273                        serde_json::from_slice(&frames[0].body).unwrap()
14274                    else {
14275                        panic!("expected a rescan response");
14276                    };
14277                    let expected = if needs_restart {
14278                        vec!["privileged_operator_prompt"]
14279                    } else {
14280                        Vec::new()
14281                    };
14282                    assert_eq!(result.restart_required, expected);
14283                    assert_eq!(handler.privileged_operator_prompt, running);
14284                    assert_eq!(
14285                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14286                        running
14287                    );
14288                }
14289            }
14290        }
14291    }
14292
14293    #[tokio::test]
14294    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14295        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14296        let path = dir.join("subc.jsonc");
14297        let registry = Arc::new(Registry::default());
14298        let handle = SupervisorHandle::new();
14299        let supervisor =
14300            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14301        let handler = ControlHandler::new(registry)
14302            .with_supervisor(handle)
14303            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14304            .with_supervisor_rescan(supervisor, &path, None);
14305        handler.forwarding.operator_confirms().configure(
14306            Arc::new(ApproveOperator),
14307            crate::operator_confirm::OperatorLimits::default(),
14308        );
14309        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14310        for preview in [true, false] {
14311            std::fs::write(
14312                &path,
14313                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14314            )
14315            .unwrap();
14316            let request = ClientControlRequest::SupervisorRescan { preview };
14317            let frame = Frame::build(
14318                FrameType::Request,
14319                control_flags(),
14320                0,
14321                0,
14322                1,
14323                serde_json::to_vec(&request).unwrap(),
14324            )
14325            .unwrap();
14326            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14327            assert_eq!(frames.len(), 1);
14328            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14329            assert_eq!(
14330                handler.privileged_operator_prompt,
14331                PrivilegedOperatorPrompt::Enforce
14332            );
14333        }
14334    }
14335
14336    /// One wire code has several senders, so the refusal line names the check
14337    /// that refused. This drives the shared refusal path for ordinary refusals
14338    /// with an unregistered
14339    /// target and requires the branch label on the event.
14340    #[tokio::test]
14341    async fn route_open_refusal_names_the_check_that_refused() {
14342        let handler = ControlHandler::new(Arc::new(Registry::default()));
14343        let capture = EventCapture::default();
14344        let _subscriber =
14345            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14346        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14347        let response = handler
14348            .handle_control_frame(
14349                &ctx,
14350                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14351            )
14352            .await
14353            .unwrap();
14354
14355        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14356        let event = capture
14357            .events()
14358            .into_iter()
14359            .find(|event| {
14360                event.target == "control"
14361                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14362            })
14363            .expect("route.open refusal event");
14364        assert_eq!(
14365            event.fields.get("reason"),
14366            Some(&"\"not_registered\"".to_string())
14367        );
14368    }
14369
14370    #[tokio::test]
14371    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14372        let registry = Arc::new(Registry::default());
14373        let supervisor_handle = SupervisorHandle::new();
14374        let supervisor =
14375            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14376                .with_handle(supervisor_handle.clone())
14377                .with_connection_file_path(std::env::temp_dir().join(format!(
14378                    "subc-route-open-refusal-info-{}",
14379                    std::process::id()
14380                )));
14381        let module = supervisor
14382            .supervise_configured(
14383                ModuleSpec {
14384                    carry_containment: false,
14385                    module_id: "warming".to_string(),
14386                    program: fake_aft_stub_path(),
14387                    args: Vec::new(),
14388                    env: Vec::new(),
14389                    reserved: false,
14390                    reserved_prefixes: Vec::new(),
14391                    protocol: ModuleProtocol::Subc,
14392                    overlap: Default::default(),
14393                },
14394                true,
14395            )
14396            .unwrap();
14397        assert_eq!(module.state().unwrap(), ModuleState::Running);
14398
14399        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14400        assert!(handler
14401            .counters()
14402            .snapshot()
14403            .get("route_open_refused_by_code")
14404            .is_none());
14405        let capture = EventCapture::default();
14406        let _subscriber =
14407            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14408        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14409        let response = handler
14410            .handle_control_frame(
14411                &ctx,
14412                route_open_frame(394, "warming", unique_project_root("refusal-info")),
14413            )
14414            .await
14415            .unwrap();
14416        module.stop().await.unwrap();
14417
14418        assert_eq!(parse_error(&response[0])["code"], "module_warming");
14419        let event = capture
14420            .events()
14421            .into_iter()
14422            .find(|event| {
14423                event.target == "control"
14424                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14425            })
14426            .expect("route.open refusal event");
14427        assert_eq!(
14428            event.fields.get("module_id"),
14429            Some(&"\"warming\"".to_string())
14430        );
14431        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14432        assert_eq!(
14433            event.fields.get("reason"),
14434            Some(&"\"supervised_not_registered\"".to_string())
14435        );
14436        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14437        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14438        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14439        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14440        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14441        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14442        assert_eq!(
14443            handler.counters().snapshot()["route_open_refused_by_code"],
14444            json!({ "module_warming": 1 })
14445        );
14446    }
14447
14448    const OUTAGE_START: &str = "route.open refusing module: not serving";
14449    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14450
14451    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14452        capture
14453            .events()
14454            .into_iter()
14455            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14456            .collect()
14457    }
14458
14459    fn supervise_stub(
14460        registry: &Arc<Registry>,
14461        module_id: &str,
14462        enabled: bool,
14463    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14464        let supervisor_handle = SupervisorHandle::new();
14465        let supervisor =
14466            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14467                .with_handle(supervisor_handle.clone())
14468                .with_connection_file_path(std::env::temp_dir().join(format!(
14469                    "subc-route-outage-{module_id}-{}",
14470                    std::process::id()
14471                )));
14472        let module = supervisor
14473            .supervise_configured(
14474                ModuleSpec {
14475                    carry_containment: false,
14476                    module_id: module_id.to_string(),
14477                    program: fake_aft_stub_path(),
14478                    args: Vec::new(),
14479                    env: Vec::new(),
14480                    reserved: false,
14481                    reserved_prefixes: Vec::new(),
14482                    protocol: ModuleProtocol::Subc,
14483                    overlap: Default::default(),
14484                },
14485                enabled,
14486            )
14487            .unwrap();
14488        (supervisor_handle, module)
14489    }
14490
14491    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14492        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14493            module_id: module_id.to_string(),
14494            drain_timeout_ms: Some(50),
14495        })
14496        .unwrap();
14497        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14498    }
14499
14500    /// Two handlers built over one forwarding table must share one outage
14501    /// tracker; separate trackers would each log their own opening line for
14502    /// the same outage.
14503    #[test]
14504    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14505        let registry = Arc::new(Registry::default());
14506        let forwarding = Arc::new(ForwardingTable::default());
14507        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14508        let second = ControlHandler::with_forwarding(registry, forwarding);
14509        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14510    }
14511
14512    /// A client can name any module id it likes. Refusing an unknown one,
14513    /// however often, must not create outage state or outage lines, or the
14514    /// tracker would be a memory sink any client could fill.
14515    #[tokio::test(flavor = "current_thread")]
14516    async fn route_open_unknown_module_refusals_add_no_outage_state() {
14517        let handler = ControlHandler::new(Arc::new(Registry::default()));
14518        let capture = EventCapture::default();
14519        let _subscriber =
14520            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14521        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14522        for corr in 0..8 {
14523            let response = handler
14524                .handle_control_frame(
14525                    &ctx,
14526                    route_open_frame(
14527                        380 + corr,
14528                        &format!("nobody-{corr}"),
14529                        unique_project_root("outage-unknown"),
14530                    ),
14531                )
14532                .await
14533                .unwrap();
14534            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14535        }
14536
14537        assert_eq!(handler.route_outages.tracked_module_count(), 0);
14538        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14539        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14540    }
14541
14542    /// Drives the refusal path end to end: a supervised module that served
14543    /// before and stopped being registered with no instruction to stop is a
14544    /// WARN, and the same module refused after an operator `supervisor.restart`
14545    /// is an INFO.
14546    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14547    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14548        let registry = Arc::new(Registry::default());
14549        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
14550        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14551        let capture = EventCapture::default();
14552        let _subscriber =
14553            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14554        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14555        // The stub never registers, so pretend it served once: otherwise every
14556        // refusal would fall in its startup window.
14557        handler.route_outages.record_accepted("outage-restart");
14558
14559        let response = handler
14560            .handle_control_frame(
14561                &ctx,
14562                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14563            )
14564            .await
14565            .unwrap();
14566        assert_eq!(response[0].header.ty, FrameType::Error);
14567        let starts = outage_lines(&capture, OUTAGE_START);
14568        assert_eq!(starts.len(), 1, "{starts:?}");
14569        assert_eq!(starts[0].level, tracing::Level::WARN);
14570        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14571        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14572        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14573        handler.route_outages.record_accepted("outage-restart");
14574        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14575
14576        let restart = handler
14577            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14578            .await
14579            .unwrap();
14580        assert_eq!(
14581            restart[0].header.ty,
14582            FrameType::Response,
14583            "{:?}",
14584            parse_error(&restart[0])
14585        );
14586        handler
14587            .handle_control_frame(
14588                &ctx,
14589                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14590            )
14591            .await
14592            .unwrap();
14593        module.stop().await.unwrap();
14594
14595        let starts = outage_lines(&capture, OUTAGE_START);
14596        assert_eq!(starts.len(), 2, "{starts:?}");
14597        assert_eq!(starts[1].level, tracing::Level::INFO);
14598        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14599    }
14600
14601    /// A restart refused before it touched the module (here: the module is
14602    /// disabled) must clear its operator mark, so the next real outage is
14603    /// still reported as a warning.
14604    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14605    async fn failed_operator_restart_leaves_no_operator_mark() {
14606        let registry = Arc::new(Registry::default());
14607        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
14608        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14609        let capture = EventCapture::default();
14610        let _subscriber =
14611            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14612        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14613        handler.route_outages.record_accepted("outage-disabled");
14614
14615        let restart = handler
14616            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14617            .await
14618            .unwrap();
14619        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14620        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14621
14622        handler
14623            .handle_control_frame(
14624                &ctx,
14625                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14626            )
14627            .await
14628            .unwrap();
14629        let starts = outage_lines(&capture, OUTAGE_START);
14630        assert_eq!(starts.len(), 1, "{starts:?}");
14631        assert_eq!(starts[0].level, tracing::Level::WARN);
14632    }
14633
14634    #[tokio::test(flavor = "current_thread")]
14635    async fn route_open_unknown_module_escapes_target_module_id() {
14636        let handler = ControlHandler::new(Arc::new(Registry::default()));
14637        let capture = EventCapture::default();
14638        let _subscriber =
14639            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14640        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14641        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14642        let response = handler
14643            .handle_control_frame(
14644                &ctx,
14645                route_open_frame(
14646                    395,
14647                    hostile_module_id,
14648                    unique_project_root("hostile-target-module-id"),
14649                ),
14650            )
14651            .await
14652            .unwrap();
14653
14654        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14655        let event = capture
14656            .events()
14657            .into_iter()
14658            .find(|event| {
14659                event.target == "control"
14660                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14661            })
14662            .expect("route.open unknown-module refusal event");
14663        let logged = event.fields.get("module_id").expect("module_id field");
14664        assert!(!logged.bytes().any(|byte| byte < 0x20));
14665        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14666    }
14667
14668    /// A route.open refusal is the daemon's own answer, so its ERROR carries
14669    /// the daemon-origin bit on the frame the client receives.
14670    #[tokio::test(flavor = "current_thread")]
14671    async fn route_open_refusal_carries_daemon_origin() {
14672        let handler = ControlHandler::new(Arc::new(Registry::default()));
14673        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14674        let response = handler
14675            .handle_control_frame(
14676                &ctx,
14677                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14678            )
14679            .await
14680            .unwrap();
14681
14682        assert_eq!(response.len(), 1);
14683        assert_eq!(response[0].header.ty, FrameType::Error);
14684        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14685        assert!(
14686            response[0].header.flags.is_daemon_origin(),
14687            "route.open refusal must carry daemon origin: {:?}",
14688            response[0].header.flags
14689        );
14690    }
14691
14692    /// The drain `module_reloading` refusal is daemon-synthesized for an
14693    /// endpoint that is reloading, so it carries the daemon-origin bit on the
14694    /// frame the client receives.
14695    #[tokio::test(flavor = "current_thread")]
14696    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14697        let registry = Arc::new(Registry::default());
14698        let forwarding = Arc::new(ForwardingTable::default());
14699        let handler =
14700            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14701        let module_connection = ConnectionId::new(95);
14702        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14703        hello_via_sink(
14704            &handler,
14705            &module_ctx,
14706            &mut module_rx,
14707            hello_frame("aft", PROTOCOL_VERSION, 395),
14708        )
14709        .await;
14710        forwarding
14711            .begin_module_drain("aft", RouteCloseReason::Restart)
14712            .unwrap()
14713            .expect("the module endpoint drains");
14714
14715        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14716        let response = handler
14717            .handle_control_frame(
14718                &client_ctx,
14719                route_open_frame(396, "aft", unique_project_root("drain-origin")),
14720            )
14721            .await
14722            .unwrap();
14723
14724        assert_eq!(response.len(), 1);
14725        assert_eq!(response[0].header.ty, FrameType::Error);
14726        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14727        assert!(
14728            response[0].header.flags.is_daemon_origin(),
14729            "drain module_reloading must carry daemon origin: {:?}",
14730            response[0].header.flags
14731        );
14732    }
14733
14734    /// A module that refuses a bind with a daemon code still cannot mint a
14735    /// daemon refusal: the relayed `module_reloading` body reaches the client
14736    /// with the daemon-origin bit clear.
14737    #[tokio::test(flavor = "current_thread")]
14738    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14739        let registry = Arc::new(Registry::default());
14740        let forwarding = Arc::new(ForwardingTable::default());
14741        let handler =
14742            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14743        let module_connection = ConnectionId::new(95);
14744        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14745        hello_via_sink(
14746            &handler,
14747            &module_ctx,
14748            &mut module_rx,
14749            hello_frame("aft", PROTOCOL_VERSION, 395),
14750        )
14751        .await;
14752
14753        let client_connection = ConnectionId::new(96);
14754        let (client_ctx, _client_rx) = route_ctx(client_connection);
14755        let (route_task, bind) = relay_route_open(
14756            &handler,
14757            client_connection,
14758            &client_ctx.egress,
14759            &mut module_rx,
14760            396,
14761            "aft",
14762            "spoofed-reloading",
14763        )
14764        .await;
14765        // The module sets the origin bit on its own rejection too: neither
14766        // the bit nor the daemon code may survive the relay as daemon-owned.
14767        let rejection = Frame::build(
14768            FrameType::Error,
14769            crate::router::daemon_error_flags(),
14770            0,
14771            0,
14772            bind.header.corr,
14773            serde_json::to_vec(&ErrorBody::new(
14774                "module_reloading",
14775                "module refused route.bind",
14776            ))
14777            .unwrap(),
14778        )
14779        .unwrap();
14780        handler
14781            .handle_control_frame(&module_ctx, rejection)
14782            .await
14783            .unwrap();
14784
14785        let response = route_task.await.unwrap();
14786        assert_eq!(response.len(), 1);
14787        assert_eq!(response[0].header.ty, FrameType::Error);
14788        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14789        assert!(
14790            !response[0].header.flags.is_daemon_origin(),
14791            "a module's own rejection must not carry daemon origin: {:?}",
14792            response[0].header.flags
14793        );
14794    }
14795
14796    #[tokio::test(flavor = "current_thread")]
14797    async fn route_open_module_rejection_uses_daemon_counter_key() {
14798        let registry = Arc::new(Registry::default());
14799        let forwarding = Arc::new(ForwardingTable::default());
14800        let handler =
14801            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14802        let module_connection = ConnectionId::new(95);
14803        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14804        hello_via_sink(
14805            &handler,
14806            &module_ctx,
14807            &mut module_rx,
14808            hello_frame("aft", PROTOCOL_VERSION, 395),
14809        )
14810        .await;
14811
14812        let client_connection = ConnectionId::new(96);
14813        let (client_ctx, _client_rx) = route_ctx(client_connection);
14814        let capture = EventCapture::default();
14815        let _subscriber =
14816            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14817        let (route_task, bind) = relay_route_open(
14818            &handler,
14819            client_connection,
14820            &client_ctx.egress,
14821            &mut module_rx,
14822            396,
14823            "aft",
14824            "hostile-module-code",
14825        )
14826        .await;
14827        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14828        let rejection = Frame::build(
14829            FrameType::Error,
14830            control_flags(),
14831            0,
14832            0,
14833            bind.header.corr,
14834            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14835        )
14836        .unwrap();
14837        handler
14838            .handle_control_frame(&module_ctx, rejection)
14839            .await
14840            .unwrap();
14841
14842        let response = route_task.await.unwrap();
14843        assert_eq!(parse_error(&response[0])["code"], hostile_code);
14844        let counters = handler.counters().snapshot();
14845        assert_eq!(
14846            counters["route_open_refused_by_code"],
14847            json!({ "module_rejected": 1 })
14848        );
14849        assert!(counters["route_open_refused_by_code"]
14850            .get(hostile_code)
14851            .is_none());
14852
14853        let event = capture
14854            .events()
14855            .into_iter()
14856            .find(|event| {
14857                event.target == "control"
14858                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14859            })
14860            .expect("route.open module-rejection refusal event");
14861        let logged = event.fields.get("module_code").expect("module_code field");
14862        assert!(!logged.bytes().any(|byte| byte < 0x20));
14863        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14864    }
14865
14866    #[tokio::test]
14867    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14868        let registry = Arc::new(Registry::default());
14869        let supervisor_handle = SupervisorHandle::new();
14870        let missing_program = std::env::temp_dir().join(format!(
14871            "subc-route-open-missing-program-{}",
14872            std::process::id()
14873        ));
14874        let supervisor =
14875            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14876                .with_handle(supervisor_handle.clone());
14877        let module = supervisor
14878            .supervise_configured(
14879                ModuleSpec {
14880                    carry_containment: false,
14881                    module_id: "failed".to_string(),
14882                    program: missing_program,
14883                    args: Vec::new(),
14884                    env: Vec::new(),
14885                    reserved: false,
14886                    reserved_prefixes: Vec::new(),
14887                    protocol: ModuleProtocol::Subc,
14888                    overlap: Default::default(),
14889                },
14890                true,
14891            )
14892            .unwrap();
14893        assert_eq!(module.state().unwrap(), ModuleState::Failed);
14894
14895        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14896        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14897        let response = handler
14898            .handle_control_frame(
14899                &ctx,
14900                route_open_frame(305, "failed", unique_project_root("failed")),
14901            )
14902            .await
14903            .unwrap();
14904
14905        assert_eq!(response[0].header.ty, FrameType::Error);
14906        let error = parse_error(&response[0]);
14907        assert_eq!(error["code"], "target_unavailable");
14908        assert!(error["message"]
14909            .as_str()
14910            .unwrap()
14911            .contains("state=failed, enabled=true, live=false"));
14912    }
14913
14914    #[tokio::test]
14915    async fn route_open_role_mismatch_remains_target_unavailable() {
14916        let registry = Arc::new(Registry::default());
14917        let handler = ControlHandler::new(Arc::clone(&registry));
14918        handler
14919            .handle_control(
14920                ConnectionId::new(41),
14921                non_routable_hello_frame_with_control_ops("health-only", 306, None),
14922            )
14923            .unwrap();
14924
14925        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14926        let response = handler
14927            .handle_control_frame(
14928                &ctx,
14929                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14930            )
14931            .await
14932            .unwrap();
14933
14934        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14935        assert!(parse_error(&response[0])["message"]
14936            .as_str()
14937            .unwrap()
14938            .contains("does not provide the requested target"));
14939    }
14940
14941    #[tokio::test]
14942    async fn route_open_inactive_registration_remains_target_unavailable() {
14943        let registry = Arc::new(Registry::default());
14944        let handler = ControlHandler::new(Arc::clone(&registry));
14945        handler
14946            .handle_control(
14947                ConnectionId::new(43),
14948                hello_frame("inactive", PROTOCOL_VERSION, 308),
14949            )
14950            .unwrap();
14951        assert!(registry
14952            .set_module_state_for_test("inactive", ChannelState::Closed)
14953            .unwrap());
14954
14955        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14956        let response = handler
14957            .handle_control_frame(
14958                &ctx,
14959                route_open_frame(309, "inactive", unique_project_root("inactive")),
14960            )
14961            .await
14962            .unwrap();
14963
14964        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14965        assert!(parse_error(&response[0])["message"]
14966            .as_str()
14967            .unwrap()
14968            .contains("is not active"));
14969    }
14970
14971    #[tokio::test]
14972    async fn late_health_reply_is_recorded_through_the_module_response_path() {
14973        let registry = Arc::new(Registry::default());
14974        let forwarding = Arc::new(ForwardingTable::default());
14975        let supervisor_handle = SupervisorHandle::new();
14976        let supervisor =
14977            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
14978                .with_forwarding(Arc::clone(&forwarding))
14979                .with_handle(supervisor_handle.clone());
14980        let module = supervisor
14981            .supervise_configured(
14982                crate::ModuleSpec {
14983                    carry_containment: false,
14984                    module_id: "late-health-response".to_string(),
14985                    program: PathBuf::from("disabled-module"),
14986                    args: Vec::new(),
14987                    env: Vec::new(),
14988                    reserved: false,
14989                    reserved_prefixes: Vec::new(),
14990                    protocol: ModuleProtocol::Subc,
14991                    overlap: Default::default(),
14992                },
14993                false,
14994            )
14995            .unwrap();
14996        let handler =
14997            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14998                .with_supervisor(supervisor_handle);
14999        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15000        handler
15001            .handle_control_frame(
15002                &module_ctx,
15003                hello_frame_with_control_ops(
15004                    "late-health-response",
15005                    PROTOCOL_VERSION,
15006                    7,
15007                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15008                ),
15009            )
15010            .await
15011            .unwrap();
15012        let probe_started_at = Instant::now() - Duration::from_millis(80);
15013        let pending = forwarding
15014            .begin_health_probe_rpc_for(
15015                "late-health-response",
15016                MODULE_CONTROL_OP_HEALTH_CHECK,
15017                probe_started_at,
15018                Instant::now() - Duration::from_millis(1),
15019            )
15020            .unwrap();
15021        assert!(forwarding
15022            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15023            .unwrap());
15024
15025        let responses = handler
15026            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15027            .await
15028            .unwrap();
15029
15030        assert!(responses.is_empty());
15031        let health = module.status().unwrap().health;
15032        assert_eq!(health.late_answer_count, 1);
15033        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15034    }
15035
15036    #[tokio::test]
15037    async fn health_probe_timeout_and_module_death_are_typed() {
15038        let registry = Arc::new(Registry::default());
15039        let forwarding = Arc::new(ForwardingTable::default());
15040        let handler =
15041            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15042                .with_health_probe_timeout(Duration::from_millis(50));
15043        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15044        hello_via_sink(
15045            &handler,
15046            &module_ctx,
15047            &mut module_rx,
15048            hello_frame_with_control_ops(
15049                "aft",
15050                PROTOCOL_VERSION,
15051                7,
15052                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15053            ),
15054        )
15055        .await;
15056
15057        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15058        let responses = handler
15059            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15060            .await
15061            .unwrap();
15062        assert_eq!(responses[0].header.ty, FrameType::Error);
15063        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15064        let _ = module_rx.try_recv();
15065
15066        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15067        let health_handler = handler.clone();
15068        let death_task = tokio::spawn(async move {
15069            health_handler
15070                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15071                .await
15072                .unwrap()
15073        });
15074        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15075            .await
15076            .unwrap()
15077            .unwrap();
15078        handler
15079            .cleanup_connection(module_ctx.connection_id)
15080            .unwrap();
15081        let responses = death_task.await.unwrap();
15082        assert_eq!(responses[0].header.ty, FrameType::Error);
15083        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15084    }
15085
15086    #[test]
15087    fn hello_requires_exact_protocol_version() {
15088        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15089            let registry = Arc::new(Registry::default());
15090            let handler = ControlHandler::new(Arc::clone(&registry));
15091            let responses = handler
15092                .handle_control(
15093                    ConnectionId::new(connection),
15094                    hello_frame("aft", offered, 9),
15095                )
15096                .unwrap();
15097
15098            assert_eq!(responses.len(), 1);
15099            assert_eq!(responses[0].header.ty, FrameType::Error);
15100            let error = parse_error(&responses[0]);
15101            assert_eq!(error["code"], "version_unsupported");
15102            assert!(registry.get_module("aft").unwrap().is_none());
15103            assert_eq!(registry.active_registration_count().unwrap(), 0);
15104        }
15105    }
15106
15107    #[test]
15108    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15109        let registry = Arc::new(Registry::default());
15110        let forwarding = Arc::new(ForwardingTable::default());
15111        let handler =
15112            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15113        let module_connection = ConnectionId::new(301);
15114        let registration = registry
15115            .register_with_control_ops(
15116                manifest("aft-push", PROTOCOL_VERSION),
15117                PROTOCOL_VERSION,
15118                module_connection,
15119                module_baseline_control_ops(),
15120            )
15121            .unwrap();
15122        let (module_tx, _module_rx) = mpsc::channel(8);
15123        let endpoint = forwarding
15124            .register_module_connection(
15125                module_connection,
15126                "aft-push".to_string(),
15127                PROTOCOL_VERSION,
15128                manifest_concurrency(&registration.manifest),
15129                FrameSink::new(module_tx),
15130            )
15131            .unwrap();
15132
15133        // A push op this version does not know is ignored (forward-compat), not errored.
15134        let unknown = Frame::build(
15135            FrameType::Push,
15136            control_flags(),
15137            0,
15138            0,
15139            5,
15140            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15141        )
15142        .unwrap();
15143        let out = handler.handle_status_update(endpoint, unknown).unwrap();
15144        assert!(
15145            out.is_empty(),
15146            "unknown push op must be ignored, got {out:?}"
15147        );
15148
15149        // A malformed body for a KNOWN op is a real error worth surfacing.
15150        let malformed = Frame::build(
15151            FrameType::Push,
15152            control_flags(),
15153            0,
15154            0,
15155            6,
15156            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15157        )
15158        .unwrap();
15159        let out = handler.handle_status_update(endpoint, malformed).unwrap();
15160        assert_eq!(out.len(), 1);
15161        assert_eq!(out[0].header.ty, FrameType::Error);
15162        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15163    }
15164
15165    #[test]
15166    fn hello_rejected_when_connection_already_owns_client_routes() {
15167        let registry = Arc::new(Registry::default());
15168        let forwarding = Arc::new(ForwardingTable::default());
15169        let handler =
15170            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15171        // Commits a client route on connection 202 (bound to a module on conn 101).
15172        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
15173        let client_connection = ConnectionId::new(202);
15174
15175        // That same connection now tries to register as a module: rejected, so one
15176        // connection never holds both client-route and module-endpoint state.
15177        let responses = handler
15178            .handle_control(
15179                client_connection,
15180                hello_frame("aft-second", PROTOCOL_VERSION, 9),
15181            )
15182            .unwrap();
15183        assert_eq!(responses[0].header.ty, FrameType::Error);
15184        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15185        assert!(registry.get_module("aft-second").unwrap().is_none());
15186    }
15187
15188    #[tokio::test]
15189    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15190        let registry = Arc::new(Registry::default());
15191        let forwarding = Arc::new(ForwardingTable::default());
15192        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15193        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15194        hello_via_sink(
15195            &handler,
15196            &module_ctx,
15197            &mut module_rx,
15198            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15199        )
15200        .await;
15201        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15202        let pending = forwarding
15203            .begin_route_bind_relay_for_test(
15204                client_ctx.connection_id,
15205                client_ctx.egress.clone(),
15206                2,
15207                "alpha",
15208            )
15209            .unwrap();
15210        forwarding
15211            .complete_pending_relay(
15212                module_ctx.connection_id,
15213                pending.corr,
15214                RouteBindRelayOutcome::Accepted,
15215            )
15216            .unwrap();
15217        client_rx.try_recv().unwrap();
15218        for module_id in ["beta", "alpha"] {
15219            let replies = handler
15220                .handle_control_frame(
15221                    &module_ctx,
15222                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15223                )
15224                .await
15225                .unwrap();
15226            assert_eq!(replies.len(), 1, "second HELLO must be refused");
15227            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15228        }
15229        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15230        assert!(registry.get_module("beta").unwrap().is_none());
15231        assert!(matches!(
15232            forwarding
15233                .lookup_data_route(
15234                    client_ctx.connection_id,
15235                    pending.client_channel,
15236                    pending.client_epoch,
15237                )
15238                .unwrap(),
15239            DataRoute::Client(DataRouteState::Bound(_))
15240        ));
15241        assert!(handler
15242            .hello_launch_nonces
15243            .lock()
15244            .unwrap()
15245            .presented(module_ctx.connection_id, Some("alpha-nonce")));
15246        assert!(module_rx.try_recv().is_err());
15247    }
15248
15249    #[test]
15250    fn reserved_module_hello_requires_matching_launch_nonce() {
15251        let registry = Arc::new(Registry::default());
15252        let supervisor = SupervisorHandle::new();
15253        // The supervisor recorded the nonce it injected when it spawned the reserved
15254        // module; the HELLO verifier checks against the same shared handle.
15255        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15256        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15257
15258        // A HELLO with NO nonce is rejected.
15259        let no_nonce = handler
15260            .handle_control(
15261                ConnectionId::new(1),
15262                hello_frame("vault", PROTOCOL_VERSION, 1),
15263            )
15264            .unwrap();
15265        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15266        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15267        assert!(registry.get_module("vault").unwrap().is_none());
15268
15269        // A HELLO with the WRONG nonce is rejected.
15270        let wrong = handler
15271            .handle_control(
15272                ConnectionId::new(2),
15273                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15274            )
15275            .unwrap();
15276        assert_eq!(wrong[0].header.ty, FrameType::Error);
15277        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15278        assert!(registry.get_module("vault").unwrap().is_none());
15279
15280        // A HELLO with the CORRECT nonce registers.
15281        let ok = handler
15282            .handle_control(
15283                ConnectionId::new(3),
15284                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15285            )
15286            .unwrap();
15287        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15288        assert!(registry.get_module("vault").unwrap().is_some());
15289    }
15290
15291    #[test]
15292    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15293        let registry = Arc::new(Registry::default());
15294        let supervisor = SupervisorHandle::new();
15295        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15296        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15297        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15298
15299        let squat = handler
15300            .handle_control(
15301                ConnectionId::new(1),
15302                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15303            )
15304            .unwrap();
15305        assert_eq!(squat[0].header.ty, FrameType::Error);
15306        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15307        assert!(parse_error(&squat[0])["message"]
15308            .as_str()
15309            .unwrap()
15310            .contains("fed:"));
15311
15312        let accepted_peer = handler
15313            .handle_control(
15314                ConnectionId::new(2),
15315                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15316            )
15317            .unwrap();
15318        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15319
15320        let accepted_short = handler
15321            .handle_control(
15322                ConnectionId::new(3),
15323                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15324            )
15325            .unwrap();
15326        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15327
15328        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15329            let response = handler
15330                .handle_control(
15331                    ConnectionId::new(conn),
15332                    hello_frame(module_id, PROTOCOL_VERSION, conn),
15333                )
15334                .unwrap();
15335            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15336        }
15337    }
15338
15339    #[test]
15340    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15341        let registry = Arc::new(Registry::default());
15342        let supervisor = SupervisorHandle::new();
15343        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15344        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15345        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15346        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15347
15348        let owner_nonce = handler
15349            .handle_control(
15350                ConnectionId::new(1),
15351                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15352            )
15353            .unwrap();
15354        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15355        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15356        assert!(registry.get_module("fed:special").unwrap().is_none());
15357
15358        let exact_nonce = handler
15359            .handle_control(
15360                ConnectionId::new(2),
15361                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15362            )
15363            .unwrap();
15364        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15365        assert!(registry.get_module("fed:special").unwrap().is_some());
15366    }
15367
15368    #[test]
15369    fn non_reserved_module_ignores_launch_nonce() {
15370        let registry = Arc::new(Registry::default());
15371        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
15372        // registration succeeds whether a spawned process echoes a nonce or not.
15373        let handler = ControlHandler::new(Arc::clone(&registry));
15374        let no_nonce = handler
15375            .handle_control(
15376                ConnectionId::new(1),
15377                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15378            )
15379            .unwrap();
15380        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15381        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15382
15383        let echoed_nonce = handler
15384            .handle_control(
15385                ConnectionId::new(2),
15386                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15387            )
15388            .unwrap();
15389        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15390        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15391    }
15392
15393    #[test]
15394    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15395        let handler = ControlHandler::default();
15396        let conn = ConnectionId::new(1);
15397        let malformed = Frame::build(
15398            FrameType::Hello,
15399            control_flags(),
15400            0,
15401            0,
15402            3,
15403            b"{not json".to_vec(),
15404        )
15405        .unwrap();
15406
15407        let error = handler.handle_control(conn, malformed).unwrap();
15408        assert_eq!(error[0].header.ty, FrameType::Error);
15409        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15410
15411        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15412        let pong = handler.handle_control(conn, ping).unwrap();
15413        assert_eq!(pong[0].header.ty, FrameType::Pong);
15414        assert_eq!(pong[0].header.corr, 4);
15415    }
15416
15417    #[test]
15418    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15419        let registry = Arc::new(Registry::default());
15420        let handler = ControlHandler::new(Arc::clone(&registry));
15421
15422        handler
15423            .handle_control(
15424                ConnectionId::new(1),
15425                hello_frame("aft", PROTOCOL_VERSION, 1),
15426            )
15427            .unwrap();
15428        let duplicate = handler
15429            .handle_control(
15430                ConnectionId::new(2),
15431                hello_frame("aft", PROTOCOL_VERSION, 2),
15432            )
15433            .unwrap();
15434
15435        assert_eq!(duplicate[0].header.ty, FrameType::Error);
15436        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15437        let registration = registry.get_module("aft").unwrap().unwrap();
15438        assert_eq!(registration.connection_id, ConnectionId::new(1));
15439    }
15440
15441    #[test]
15442    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15443        let registry = Arc::new(Registry::default());
15444        let forwarding = Arc::new(ForwardingTable::default());
15445        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15446        let handler =
15447            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15448                .with_process_liveness(process_liveness);
15449        let (ctx, route_channel, route_epoch) =
15450            bind_liveness_route(&registry, &forwarding, "aft-dead");
15451        let responses = handler
15452            .handle_route_poll(
15453                &ctx,
15454                route_poll_frame(41, PollKind::Liveness, route_channel),
15455                route_channel,
15456                route_epoch,
15457                PollKind::Liveness,
15458            )
15459            .unwrap();
15460
15461        assert_eq!(responses.len(), 1);
15462        assert_eq!(responses[0].header.ty, FrameType::Response);
15463        assert_route_poll_liveness(&responses[0], false);
15464    }
15465
15466    #[test]
15467    fn liveness_poll_without_process_source_uses_bound_route() {
15468        let registry = Arc::new(Registry::default());
15469        let forwarding = Arc::new(ForwardingTable::default());
15470        let handler =
15471            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15472        let (ctx, route_channel, route_epoch) =
15473            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
15474        let responses = handler
15475            .handle_route_poll(
15476                &ctx,
15477                route_poll_frame(42, PollKind::Liveness, route_channel),
15478                route_channel,
15479                route_epoch,
15480                PollKind::Liveness,
15481            )
15482            .unwrap();
15483
15484        assert_route_poll_liveness(&responses[0], true);
15485    }
15486
15487    #[test]
15488    fn liveness_poll_untracked_process_source_uses_bound_route() {
15489        let registry = Arc::new(Registry::default());
15490        let forwarding = Arc::new(ForwardingTable::default());
15491        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15492        let handler =
15493            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15494                .with_process_liveness(process_liveness);
15495        let (ctx, route_channel, route_epoch) =
15496            bind_liveness_route(&registry, &forwarding, "aft-untracked");
15497        let responses = handler
15498            .handle_route_poll(
15499                &ctx,
15500                route_poll_frame(43, PollKind::Liveness, route_channel),
15501                route_channel,
15502                route_epoch,
15503                PollKind::Liveness,
15504            )
15505            .unwrap();
15506
15507        assert_route_poll_liveness(&responses[0], true);
15508    }
15509
15510    #[tokio::test]
15511    async fn unknown_op_returns_unknown_control_op() {
15512        let handler = ControlHandler::default();
15513        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15514        let request = Frame::build(
15515            FrameType::Request,
15516            control_flags(),
15517            0,
15518            0,
15519            55,
15520            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15521        )
15522        .unwrap();
15523
15524        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15525
15526        assert_eq!(response.len(), 1);
15527        assert_eq!(response[0].header.ty, FrameType::Error);
15528        assert_eq!(response[0].header.corr, 55);
15529        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15530    }
15531
15532    #[tokio::test]
15533    async fn supervisor_provenance_rejects_unknown_exact_module() {
15534        let handler = ControlHandler::default();
15535        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15536        let request = Frame::build(
15537            FrameType::Request,
15538            control_flags(),
15539            0,
15540            0,
15541            57,
15542            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15543        )
15544        .unwrap();
15545
15546        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15547
15548        assert_eq!(response.len(), 1);
15549        assert_eq!(response[0].header.ty, FrameType::Error);
15550        assert_eq!(response[0].header.corr, 57);
15551        let error = parse_error(&response[0]);
15552        assert_eq!(error["code"], "unknown_module");
15553        assert_eq!(error["message"], "module_id 'missing' is not supervised");
15554    }
15555
15556    #[test]
15557    fn provenance_probe_override_keeps_handler_tests_deterministic() {
15558        let expected = subc_control::RunningImageAgreement::Unavailable {
15559            reason: subc_control::RunningImageUnavailableReason::HashFailed,
15560        };
15561        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15562        assert_eq!(handler.provenance_probe_override, Some(expected));
15563    }
15564
15565    #[test]
15566    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15567        let verdict = reload_verdict(
15568            std::path::Path::new("/bin/new"),
15569            Some(std::path::Path::new("/bin/old")),
15570            subc_control::RunningImageAgreement::Unavailable {
15571                reason: subc_control::RunningImageUnavailableReason::HashFailed,
15572            },
15573        );
15574        assert!(matches!(
15575            verdict.path,
15576            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15577                if configured == std::path::Path::new("/bin/new")
15578                    && spawned_from == std::path::Path::new("/bin/old")
15579        ));
15580    }
15581
15582    #[test]
15583    fn launch_spec_without_running_digest_is_unavailable_not_match() {
15584        use subc_control::{
15585            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15586        };
15587        let configured = crate::supervise::ModuleSpec {
15588            carry_containment: false,
15589            module_id: "legacy".into(),
15590            program: "/module".into(),
15591            args: Vec::new(),
15592            env: Vec::new(),
15593            reserved: false,
15594            reserved_prefixes: Vec::new(),
15595            protocol: subc_control::ModuleProtocol::Subc,
15596            overlap: Default::default(),
15597        };
15598        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15599        assert_eq!(
15600            launch_spec_agreement(&configured, false, Some(&digest)),
15601            Agreement::Unavailable {
15602                reason: Reason::NotRunning
15603            }
15604        );
15605        assert_eq!(
15606            launch_spec_agreement(&configured, true, None),
15607            Agreement::Unavailable {
15608                reason: Reason::SpawnedSpecUnavailable
15609            }
15610        );
15611        assert_eq!(
15612            launch_spec_agreement(&configured, true, Some(&digest)),
15613            Agreement::Match
15614        );
15615    }
15616
15617    #[cfg(windows)]
15618    #[test]
15619    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15620        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15621        let configured = dir.join("module.exe");
15622        std::fs::write(&configured, b"executable path fixture").unwrap();
15623        let spawned = std::fs::canonicalize(&configured).unwrap();
15624        assert_ne!(
15625            configured, spawned,
15626            "fixture must exercise different path spellings"
15627        );
15628        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15629        let lower_case = configured.to_string_lossy().to_lowercase();
15630        for spelling in [
15631            &configured,
15632            Path::new(&forward_slashes),
15633            Path::new(&lower_case),
15634        ] {
15635            let verdict = reload_verdict(
15636                spelling,
15637                Some(&spawned),
15638                subc_control::RunningImageAgreement::Unavailable {
15639                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15640                },
15641            );
15642            assert_eq!(
15643                verdict.path,
15644                ReloadPathAgreement::Match,
15645                "configured {spelling:?}, spawned {spawned:?}"
15646            );
15647        }
15648    }
15649
15650    #[cfg(windows)]
15651    #[test]
15652    fn windows_reload_path_keeps_different_existing_program_pending() {
15653        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15654        let old = dir.join("old.exe");
15655        let new = dir.join("new.exe");
15656        std::fs::write(&old, b"same executable content").unwrap();
15657        std::fs::copy(&old, &new).unwrap();
15658        let spawned = std::fs::canonicalize(&old).unwrap();
15659        let image = subc_control::RunningImageAgreement::Unavailable {
15660            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15661        };
15662        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15663        assert!(
15664            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15665            if configured == new && spawned_from == spawned)
15666        );
15667        let missing = dir.join("missing.exe");
15668        assert!(matches!(
15669            reload_verdict(&missing, Some(&spawned), image).path,
15670            ReloadPathAgreement::Mismatch { .. }
15671        ));
15672    }
15673
15674    #[test]
15675    fn reload_verdict_detects_replaced_image_at_same_path() {
15676        let image = subc_control::RunningImageAgreement::Mismatch {
15677            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15678                digest: "old".into(),
15679            },
15680            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15681                digest: "new".into(),
15682            },
15683        };
15684        let verdict = reload_verdict(
15685            std::path::Path::new("/bin/same"),
15686            Some(std::path::Path::new("/bin/same")),
15687            image.clone(),
15688        );
15689        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15690        assert_eq!(verdict.image, image);
15691    }
15692
15693    #[test]
15694    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15695        let image = subc_control::RunningImageAgreement::Unavailable {
15696            reason: subc_control::RunningImageUnavailableReason::NotRunning,
15697        };
15698        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15699        assert_eq!(
15700            verdict.path,
15701            subc_control::ReloadPathAgreement::Unavailable {
15702                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15703            }
15704        );
15705        assert_eq!(verdict.image, image);
15706
15707        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15708            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15709        };
15710        let verdict = reload_verdict(
15711            std::path::Path::new("/bin/same"),
15712            Some(std::path::Path::new("/bin/same")),
15713            unconfirmed.clone(),
15714        );
15715        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15716        assert_eq!(verdict.image, unconfirmed);
15717    }
15718
15719    #[test]
15720    fn reload_verdict_preserves_each_image_unavailability_reason() {
15721        use subc_control::RunningImageUnavailableReason as Reason;
15722
15723        for reason in [
15724            Reason::NotRunning,
15725            Reason::UnsupportedPlatform,
15726            Reason::RunningExecutableUnreadable,
15727            Reason::SpawnedPathUnreadable,
15728            Reason::HashFailed,
15729            Reason::ProcessIdentityUnconfirmed,
15730            Reason::Unknown("future_probe_reason".to_string()),
15731        ] {
15732            let image = subc_control::RunningImageAgreement::Unavailable {
15733                reason: reason.clone(),
15734            };
15735            let verdict = reload_verdict(
15736                std::path::Path::new("/bin/same"),
15737                Some(std::path::Path::new("/bin/same")),
15738                image.clone(),
15739            );
15740            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15741            assert_eq!(verdict.image, image, "{reason:?}");
15742        }
15743    }
15744
15745    #[tokio::test]
15746    async fn malformed_control_bodies_return_invalid_control_body() {
15747        let handler = ControlHandler::default();
15748        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15749
15750        for (corr, body) in [
15751            (56, br#"{"route_channel":1}"#.as_slice()),
15752            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15753            (
15754                58,
15755                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15756            ),
15757        ] {
15758            let request = Frame::build(
15759                FrameType::Request,
15760                control_flags(),
15761                0,
15762                0,
15763                corr,
15764                body.to_vec(),
15765            )
15766            .unwrap();
15767            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15768
15769            assert_eq!(response.len(), 1);
15770            assert_eq!(response[0].header.ty, FrameType::Error);
15771            assert_eq!(response[0].header.corr, corr);
15772            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15773        }
15774    }
15775
15776    #[tokio::test]
15777    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15778        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15779        let registry = Arc::new(Registry::default());
15780        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15781        let router = Router::with_control_handler(Arc::clone(&control));
15782        let connection = router.begin_connection();
15783        let (ctx, mut rx) = route_ctx(connection.id());
15784
15785        router
15786            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15787            .await
15788            .unwrap();
15789        let response = rx.recv().await.unwrap();
15790        let ack = parse_ack(&response);
15791        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15792        let channel = 1;
15793
15794        let goodbye =
15795            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15796        router.route_for_connection(&ctx, goodbye).await.unwrap();
15797        assert!(rx.try_recv().is_err());
15798        assert!(registry.get_module("aft").unwrap().is_none());
15799
15800        router
15801            .route_for_connection(&ctx, channel_request(channel, 13))
15802            .await
15803            .unwrap();
15804        let error_frame = rx.recv().await.unwrap();
15805        assert_eq!(error_frame.header.ty, FrameType::Error);
15806        assert_eq!(error_frame.header.channel, channel);
15807        let captured = crate::router::test_log::captured_logs(&logs);
15808        assert_eq!(
15809            captured
15810                .lines()
15811                .filter(|line| {
15812                    line.contains("module_id=aft")
15813                        && line.contains("reason=explicit_goodbye")
15814                        && line.contains("module registration ended")
15815                })
15816                .count(),
15817            1,
15818            "unexpected GOODBYE registry log: {captured}"
15819        );
15820    }
15821
15822    #[tokio::test]
15823    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15824        let registry = Arc::new(Registry::default());
15825        let handler = ControlHandler::new(registry).with_capability_config(
15826            [("prov".to_string(), true), ("cons".to_string(), true)],
15827            BTreeMap::new(),
15828        );
15829        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15830        register_capability_manifest(
15831            &handler,
15832            &provider_ctx,
15833            &mut provider_rx,
15834            capability_manifest("prov", &["thing/v1"], &[]),
15835            1,
15836        )
15837        .await;
15838        let mut consumer = capability_manifest("cons", &[], &[]);
15839        consumer.capabilities.as_mut().unwrap().requires.push(
15840            subc_protocol::manifest::CapabilityRequirement {
15841                capability: "thing/v1".to_string(),
15842                need: subc_protocol::manifest::CapabilityNeed::Required,
15843            },
15844        );
15845        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15846        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15847        assert_eq!(
15848            handler.capability_evaluator.verdict("cons", "thing/v1"),
15849            Some(CapabilityVerdict::Provided)
15850        );
15851        let (mut client_rx, _) = open_route_for_capability_test(
15852            &handler,
15853            &provider_ctx,
15854            &mut provider_rx,
15855            703,
15856            3,
15857            "prov",
15858            None,
15859        )
15860        .await;
15861        let goodbye =
15862            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15863        handler
15864            .handle_control_frame(&provider_ctx, goodbye)
15865            .await
15866            .unwrap();
15867        assert_eq!(
15868            handler.capability_evaluator.verdict("cons", "thing/v1"),
15869            Some(CapabilityVerdict::NeverProvided)
15870        );
15871        let closed = client_rx
15872            .try_recv()
15873            .expect("GOODBYE pushes route.closed before route GOODBYE");
15874        assert!(
15875            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15876            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15877        );
15878        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15879        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15880    }
15881
15882    struct RouteClosedFixture {
15883        handler: ControlHandler,
15884        #[cfg(unix)]
15885        supervisor: Supervisor,
15886        module_ctx: RouteCtx,
15887        #[cfg(unix)]
15888        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15889        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15890        route: Arc<crate::forwarding::RouteBinding>,
15891    }
15892
15893    async fn route_closed_fixture() -> RouteClosedFixture {
15894        let registry = Arc::new(Registry::default());
15895        let handler = ControlHandler::new(Arc::clone(&registry));
15896        #[cfg(unix)]
15897        let supervisor = Supervisor::new(registry, RestartPolicy::default())
15898            .with_forwarding(Arc::clone(&handler.forwarding));
15899        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15900        register_capability_manifest(
15901            &handler,
15902            &module_ctx,
15903            &mut module_rx,
15904            capability_manifest("prov", &[], &[]),
15905            1,
15906        )
15907        .await;
15908        let (client_rx, bind) = open_route_for_capability_test(
15909            &handler,
15910            &module_ctx,
15911            &mut module_rx,
15912            702,
15913            2,
15914            "prov",
15915            None,
15916        )
15917        .await;
15918        let ModuleControlRequest::RouteBind {
15919            route_channel: channel,
15920            epoch: route_epoch,
15921            ..
15922        } = bind
15923        else {
15924            panic!("expected route.bind");
15925        };
15926        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15927            handler
15928                .forwarding
15929                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15930                .unwrap()
15931        else {
15932            panic!("the acknowledged route must be bound");
15933        };
15934        RouteClosedFixture {
15935            handler,
15936            #[cfg(unix)]
15937            supervisor,
15938            module_ctx,
15939            #[cfg(unix)]
15940            module_rx,
15941            client_rx,
15942            route,
15943        }
15944    }
15945
15946    impl RouteClosedFixture {
15947        #[cfg(unix)]
15948        async fn cut(&mut self) {
15949            let notices = async {
15950                let mut draining = self.module_rx.recv().await.unwrap();
15951                assert!(matches!(
15952                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15953                        &draining.body
15954                    )
15955                    .unwrap(),
15956                    subc_protocol::session::ModuleControlCommand::Draining {
15957                        reason: RouteCloseReason::Disable,
15958                        ..
15959                    }
15960                ));
15961                draining.flushed.take().unwrap().send(()).unwrap();
15962                let mut closing = self.client_rx.recv().await.unwrap();
15963                assert!(matches!(
15964                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15965                    ClientControlPush::RouteClosing {
15966                        reason: RouteCloseReason::Restart,
15967                        ..
15968                    }
15969                ));
15970                closing.flushed.take().unwrap().send(()).unwrap();
15971            };
15972            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15973            result.unwrap();
15974        }
15975
15976        fn disconnect(&mut self) -> ClientControlPush {
15977            self.handler
15978                .cleanup_connection(self.module_ctx.connection_id)
15979                .unwrap();
15980            let closed = self.client_rx.try_recv().unwrap();
15981            assert_eq!(closed.header.ty, FrameType::Push);
15982            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15983            assert!(matches!(
15984                &push,
15985                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15986                    if channels == &[self.route.client_channel]
15987            ));
15988            assert_eq!(
15989                self.client_rx.try_recv().unwrap().header.ty,
15990                FrameType::Goodbye
15991            );
15992            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15993            push
15994        }
15995    }
15996
15997    #[cfg(unix)]
15998    #[tokio::test(start_paused = true)]
15999    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16000        let mut fixture = route_closed_fixture().await;
16001        fixture.cut().await;
16002        assert!(matches!(
16003            fixture.disconnect(),
16004            ClientControlPush::RouteClosed {
16005                reason: RouteCloseReason::Restart,
16006                drained: true,
16007                ..
16008            }
16009        ));
16010    }
16011
16012    #[cfg(unix)]
16013    #[tokio::test(start_paused = true)]
16014    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16015        let mut fixture = route_closed_fixture().await;
16016        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16017        let started = tokio::time::Instant::now();
16018        fixture.cut().await;
16019        assert!(started.elapsed() >= Duration::from_secs(2));
16020        assert_eq!(fixture.route.flow.in_flight(), 1);
16021        assert!(matches!(
16022            fixture.disconnect(),
16023            ClientControlPush::RouteClosed {
16024                reason: RouteCloseReason::Restart,
16025                drained: false,
16026                ..
16027            }
16028        ));
16029    }
16030
16031    #[tokio::test]
16032    async fn module_connection_crash_reports_crash_and_not_drained() {
16033        let mut fixture = route_closed_fixture().await;
16034        // The server calls cleanup_connection on module EOF. Without a planned
16035        // daemon shutdown, that disconnect must remain a crash with no prior
16036        // route.closing notice, even when the route has no in-flight calls.
16037        assert!(matches!(
16038            fixture.disconnect(),
16039            ClientControlPush::RouteClosed {
16040                reason: RouteCloseReason::Crash,
16041                drained: false,
16042                ..
16043            }
16044        ));
16045    }
16046
16047    #[tokio::test]
16048    async fn dropping_router_connection_releases_registration() {
16049        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16050        let registry = Arc::new(Registry::default());
16051        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16052        let router = Router::with_control_handler(Arc::clone(&control));
16053        let connection = router.begin_connection();
16054        let connection_id = connection.id();
16055        let (ctx, mut rx) = route_ctx(connection_id);
16056
16057        router
16058            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16059            .await
16060            .unwrap();
16061        let response = rx.recv().await.unwrap();
16062        let ack = parse_ack(&response);
16063        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16064        assert!(registry.get_module("aft").unwrap().is_some());
16065
16066        drop(connection);
16067
16068        assert!(registry.get_module("aft").unwrap().is_none());
16069        assert_eq!(registry.active_registration_count().unwrap(), 0);
16070
16071        control
16072            .cleanup_connection(ConnectionId::new(u64::MAX))
16073            .unwrap();
16074        let captured = crate::router::test_log::captured_logs(&logs);
16075        println!("captured registry lifecycle logs:\n{captured}");
16076        let events: Vec<_> = captured
16077            .lines()
16078            .filter(|line| {
16079                line.contains("module registered module_id=aft ")
16080                    || line.contains("module registration ended")
16081            })
16082            .collect();
16083        assert_eq!(
16084            events.len(),
16085            2,
16086            "unexpected registry lifecycle logs: {captured}"
16087        );
16088        assert!(events[0].contains("module registered module_id=aft "));
16089        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16090        assert!(events[1].contains(&format!(
16091            "module_id=aft connection_id={}",
16092            connection_id.get()
16093        )));
16094        assert!(events[1].contains("reason=connection_closed"));
16095        assert!(events[1].contains("module registration ended"));
16096        assert_eq!(
16097            captured
16098                .lines()
16099                .filter(|line| line.contains("module registered module_id=aft "))
16100                .count(),
16101            1,
16102            "legacy registration admission line must appear once: {captured}"
16103        );
16104    }
16105
16106    #[tokio::test]
16107    async fn hello_registration_keeps_legacy_module_registered_line_once() {
16108        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16109        let registry = Arc::new(Registry::default());
16110        let control = ControlHandler::new(Arc::clone(&registry));
16111        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16112        hello_via_sink(
16113            &control,
16114            &ctx,
16115            &mut rx,
16116            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16117        )
16118        .await;
16119
16120        let captured = crate::router::test_log::captured_logs(&logs);
16121        let admissions: Vec<_> = captured
16122            .lines()
16123            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16124            .collect();
16125        assert_eq!(
16126            admissions.len(),
16127            1,
16128            "legacy admission line must remain exactly once: {captured}"
16129        );
16130        assert!(admissions[0].contains("routable_provider=true"));
16131        assert!(admissions[0].contains("connection_id=777"));
16132    }
16133
16134    fn capability_manifest(
16135        module_id: &str,
16136        provides: &[&str],
16137        must_never_reach: &[&str],
16138    ) -> ModuleManifest {
16139        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16140        manifest.capabilities = Some(CapabilityDeclarations {
16141            provides: provides
16142                .iter()
16143                .map(|capability| (*capability).to_string())
16144                .collect(),
16145            requires: Vec::new(),
16146            must_never_reach: must_never_reach
16147                .iter()
16148                .map(|capability| (*capability).to_string())
16149                .collect(),
16150        });
16151        manifest
16152    }
16153
16154    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16155        Frame::build(
16156            FrameType::Hello,
16157            control_flags(),
16158            0,
16159            0,
16160            corr,
16161            serde_json::to_vec(&ModuleHelloBody {
16162                protocol_ver: manifest.protocol_ver,
16163                manifest,
16164                control_ops: None,
16165                launch_nonce: None,
16166            })
16167            .expect("capability test HELLO serializes"),
16168        )
16169        .expect("capability test HELLO frame builds")
16170    }
16171
16172    fn catalog_update_with_capabilities_frame(
16173        corr: u64,
16174        capabilities: CapabilityDeclarations,
16175    ) -> Frame {
16176        Frame::build(
16177            FrameType::Request,
16178            control_flags(),
16179            0,
16180            0,
16181            corr,
16182            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16183                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16184                capabilities: Some(capabilities),
16185                ready: None,
16186            })
16187            .expect("capability catalog.update serializes"),
16188        )
16189        .expect("capability catalog.update frame builds")
16190    }
16191
16192    async fn register_capability_manifest(
16193        handler: &ControlHandler,
16194        ctx: &RouteCtx,
16195        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16196        manifest: ModuleManifest,
16197        corr: u64,
16198    ) {
16199        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16200    }
16201
16202    async fn open_route_for_capability_test(
16203        handler: &ControlHandler,
16204        target_ctx: &RouteCtx,
16205        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16206        client_connection_id: u64,
16207        corr: u64,
16208        target_module_id: &str,
16209        consumer_identity: Option<ConsumerIdentity>,
16210    ) -> (
16211        mpsc::Receiver<crate::router::OutboundFrame>,
16212        ModuleControlRequest,
16213    ) {
16214        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16215        let route_handler = handler.clone();
16216        let target_module_id = target_module_id.to_string();
16217        let route_task = tokio::spawn(async move {
16218            route_handler
16219                .handle_control_frame(
16220                    &client_ctx,
16221                    route_open_frame_with_admission_facts(
16222                        corr,
16223                        &target_module_id,
16224                        unique_project_root("admission-facts"),
16225                        consumer_identity,
16226                        None,
16227                    ),
16228                )
16229                .await
16230                .expect("capability test route.open succeeds")
16231        });
16232        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16233            .await
16234            .expect("capability test route.open must reach route.bind")
16235            .expect("target control receiver stays open");
16236        let bind_request: ModuleControlRequest =
16237            serde_json::from_slice(&bind.body).expect("route.bind decodes");
16238        handler
16239            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16240            .await
16241            .expect("capability test route.bind ACK succeeds");
16242        assert!(route_task.await.expect("route.open task joins").is_empty());
16243        let opened = client_rx
16244            .recv()
16245            .await
16246            .expect("successful route.open publishes a response");
16247        assert!(matches!(
16248            serde_json::from_slice::<ClientControlResponse>(&opened.body),
16249            Ok(ClientControlResponse::RouteOpen { .. })
16250        ));
16251        (client_rx, bind_request)
16252    }
16253
16254    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16255        assert_eq!(frame.header.ty, FrameType::Push);
16256        assert_eq!(frame.header.channel, 0);
16257        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16258            .expect("route.closed control push decodes");
16259        let ClientControlPush::RouteClosed { channels, .. } = &push else {
16260            panic!("expected route.closed");
16261        };
16262        assert_eq!(channels.len(), 1, "exactly one violating route closed");
16263        let closed_channel = channels[0];
16264        let channels = channels.clone();
16265        assert_eq!(
16266            push,
16267            ClientControlPush::RouteClosed {
16268                module_id: target_module_id.to_string(),
16269                channels,
16270                reason: RouteCloseReason::CapabilityDenied,
16271                drained: false,
16272                abandoned: 0,
16273                excluded_subscriptions: 0,
16274                terminal: Some(false),
16275            }
16276        );
16277        closed_channel
16278    }
16279
16280    #[tokio::test]
16281    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16282        let registry = Arc::new(Registry::default());
16283        let forwarding = Arc::new(ForwardingTable::default());
16284        let supervisor = SupervisorHandle::new();
16285        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16286        let handler =
16287            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16288                .with_supervisor(supervisor);
16289        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16290        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16291        register_capability_manifest(
16292            &handler,
16293            &target_ctx,
16294            &mut target_rx,
16295            capability_manifest("target", &["credentials-provider/v1"], &[]),
16296            1,
16297        )
16298        .await;
16299        register_capability_manifest(
16300            &handler,
16301            &opener_ctx,
16302            &mut opener_rx,
16303            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16304            2,
16305        )
16306        .await;
16307
16308        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16309        let replies = handler
16310            .handle_control_frame(
16311                &client_ctx,
16312                route_open_frame_with_admission_facts(
16313                    3,
16314                    "target",
16315                    unique_project_root("admission-facts"),
16316                    Some(ConsumerIdentity {
16317                        module_id: "opener".to_string(),
16318                        launch_nonce: "opener-nonce".to_string(),
16319                    }),
16320                    None,
16321                ),
16322            )
16323            .await
16324            .expect("denied route.open returns a typed frame");
16325        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16326        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16327        assert!(
16328            target_rx.try_recv().is_err(),
16329            "forbidden route.open must not relay route.bind"
16330        );
16331    }
16332
16333    #[tokio::test]
16334    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16335        let registry = Arc::new(Registry::default());
16336        let forwarding = Arc::new(ForwardingTable::default());
16337        let supervisor = SupervisorHandle::new();
16338        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16339        let handler =
16340            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16341                .with_supervisor(supervisor);
16342        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16343        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16344        register_capability_manifest(
16345            &handler,
16346            &target_ctx,
16347            &mut target_rx,
16348            capability_manifest("target", &["credentials-provider/v1"], &[]),
16349            1,
16350        )
16351        .await;
16352        register_capability_manifest(
16353            &handler,
16354            &old_opener_ctx,
16355            &mut old_opener_rx,
16356            capability_manifest("opener", &[], &[]),
16357            2,
16358        )
16359        .await;
16360        let (mut client_rx, _) = open_route_for_capability_test(
16361            &handler,
16362            &target_ctx,
16363            &mut target_rx,
16364            712,
16365            3,
16366            "target",
16367            Some(ConsumerIdentity {
16368                module_id: "opener".to_string(),
16369                launch_nonce: "opener-nonce".to_string(),
16370            }),
16371        )
16372        .await;
16373        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16374
16375        handler
16376            .cleanup_connection(old_opener_ctx.connection_id)
16377            .expect("old opener registration cleans up");
16378        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16379        register_capability_manifest(
16380            &handler,
16381            &new_opener_ctx,
16382            &mut new_opener_rx,
16383            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16384            4,
16385        )
16386        .await;
16387
16388        let closed_channel = assert_capability_denied_push(
16389            client_rx
16390                .try_recv()
16391                .expect("HELLO deny addition must emit route.closed")
16392                .frame,
16393            "target",
16394        );
16395        let goodbye = client_rx
16396            .try_recv()
16397            .expect("HELLO deny addition must send the client a route GOODBYE")
16398            .frame;
16399        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16400        assert_eq!(goodbye.header.channel, closed_channel);
16401        assert_ne!(goodbye.header.channel, 0);
16402        assert_ne!(goodbye.header.epoch, 0);
16403        assert!(client_rx.try_recv().is_err());
16404        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16405        assert!(matches!(
16406            target_rx.try_recv(),
16407            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16408        ));
16409    }
16410
16411    #[tokio::test]
16412    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16413        let registry = Arc::new(Registry::default());
16414        let forwarding = Arc::new(ForwardingTable::default());
16415        let supervisor = SupervisorHandle::new();
16416        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16417        let handler =
16418            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16419                .with_supervisor(supervisor);
16420        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16421        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16422        register_capability_manifest(
16423            &handler,
16424            &target_ctx,
16425            &mut target_rx,
16426            capability_manifest("target", &[], &[]),
16427            1,
16428        )
16429        .await;
16430        register_capability_manifest(
16431            &handler,
16432            &opener_ctx,
16433            &mut opener_rx,
16434            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16435            2,
16436        )
16437        .await;
16438        let (mut client_rx, _) = open_route_for_capability_test(
16439            &handler,
16440            &target_ctx,
16441            &mut target_rx,
16442            722,
16443            3,
16444            "target",
16445            Some(ConsumerIdentity {
16446                module_id: "opener".to_string(),
16447                launch_nonce: "opener-nonce".to_string(),
16448            }),
16449        )
16450        .await;
16451        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16452
16453        let replies = handler
16454            .handle_control_frame(
16455                &target_ctx,
16456                catalog_update_with_capabilities_frame(
16457                    4,
16458                    CapabilityDeclarations {
16459                        provides: vec!["credentials-provider/v1".to_string()],
16460                        requires: Vec::new(),
16461                        must_never_reach: Vec::new(),
16462                    },
16463                ),
16464            )
16465            .await
16466            .expect("claim catalog.update succeeds");
16467        assert!(matches!(
16468            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16469            Ok(ModuleControlResponseToModule::CatalogUpdate {})
16470        ));
16471        let closed_channel = assert_capability_denied_push(
16472            client_rx
16473                .try_recv()
16474                .expect("claim addition must emit route.closed")
16475                .frame,
16476            "target",
16477        );
16478        let goodbye = client_rx
16479            .try_recv()
16480            .expect("claim addition must send the client a route GOODBYE")
16481            .frame;
16482        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16483        assert_eq!(goodbye.header.channel, closed_channel);
16484        assert_ne!(goodbye.header.channel, 0);
16485        assert_ne!(goodbye.header.epoch, 0);
16486        assert!(client_rx.try_recv().is_err());
16487        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16488        assert!(matches!(
16489            target_rx.try_recv(),
16490            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16491        ));
16492    }
16493
16494    #[tokio::test]
16495    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16496        let registry = Arc::new(Registry::default());
16497        let forwarding = Arc::new(ForwardingTable::default());
16498        let supervisor = SupervisorHandle::new();
16499        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16500        let handler =
16501            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16502                .with_supervisor(supervisor);
16503        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16504        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16505        register_capability_manifest(
16506            &handler,
16507            &target_ctx,
16508            &mut target_rx,
16509            capability_manifest("target", &["credentials-provider/v1"], &[]),
16510            1,
16511        )
16512        .await;
16513        register_capability_manifest(
16514            &handler,
16515            &opener_ctx,
16516            &mut opener_rx,
16517            capability_manifest("opener", &[], &[]),
16518            2,
16519        )
16520        .await;
16521        let (mut client_rx, _) = open_route_for_capability_test(
16522            &handler,
16523            &target_ctx,
16524            &mut target_rx,
16525            732,
16526            3,
16527            "target",
16528            Some(ConsumerIdentity {
16529                module_id: "opener".to_string(),
16530                launch_nonce: "opener-nonce".to_string(),
16531            }),
16532        )
16533        .await;
16534        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16535
16536        handler
16537            .handle_control_frame(
16538                &target_ctx,
16539                catalog_update_with_capabilities_frame(
16540                    4,
16541                    CapabilityDeclarations {
16542                        provides: Vec::new(),
16543                        requires: Vec::new(),
16544                        must_never_reach: Vec::new(),
16545                    },
16546                ),
16547            )
16548            .await
16549            .expect("claim removal catalog.update succeeds");
16550        assert_eq!(
16551            forwarding.active_binding_count().unwrap(),
16552            1,
16553            "removing an attested target claim must leave the route census unchanged"
16554        );
16555        assert!(
16556            client_rx.try_recv().is_err(),
16557            "claim removal must not emit route.closed or a client route GOODBYE"
16558        );
16559        assert!(
16560            target_rx.try_recv().is_err(),
16561            "claim removal must not send the target a route GOODBYE"
16562        );
16563    }
16564
16565    /// A direct client may open a route to a denied capability provider; this
16566    /// policy applies only to attested supervised module origins, not to direct clients.
16567    #[tokio::test]
16568    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16569        let registry = Arc::new(Registry::default());
16570        let forwarding = Arc::new(ForwardingTable::default());
16571        let supervisor = SupervisorHandle::new();
16572        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16573        let handler =
16574            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16575                .with_supervisor(supervisor);
16576        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16577        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16578        register_capability_manifest(
16579            &handler,
16580            &target_ctx,
16581            &mut target_rx,
16582            capability_manifest("target", &["credentials-provider/v1"], &[]),
16583            1,
16584        )
16585        .await;
16586        register_capability_manifest(
16587            &handler,
16588            &opener_ctx,
16589            &mut opener_rx,
16590            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16591            2,
16592        )
16593        .await;
16594
16595        let (_client_rx, bind) = open_route_for_capability_test(
16596            &handler,
16597            &target_ctx,
16598            &mut target_rx,
16599            742,
16600            3,
16601            "target",
16602            None,
16603        )
16604        .await;
16605        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16606            panic!("direct scope-honesty route must bind");
16607        };
16608        assert_eq!(principal, Some(Principal::Direct));
16609        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16610    }
16611
16612    /// A module that denies a capability receives no self-route exemption when it
16613    /// also attestedly provides that capability.
16614    #[tokio::test]
16615    async fn must_never_reach_self_route_is_capability_forbidden() {
16616        let registry = Arc::new(Registry::default());
16617        let forwarding = Arc::new(ForwardingTable::default());
16618        let supervisor = SupervisorHandle::new();
16619        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16620        let handler =
16621            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16622                .with_supervisor(supervisor);
16623        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16624        register_capability_manifest(
16625            &handler,
16626            &self_ctx,
16627            &mut self_rx,
16628            capability_manifest(
16629                "self-provider",
16630                &["credentials-provider/v1"],
16631                &["credentials-provider/v1"],
16632            ),
16633            1,
16634        )
16635        .await;
16636
16637        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16638        let replies = handler
16639            .handle_control_frame(
16640                &client_ctx,
16641                route_open_frame_with_admission_facts(
16642                    2,
16643                    "self-provider",
16644                    unique_project_root("admission-facts"),
16645                    Some(ConsumerIdentity {
16646                        module_id: "self-provider".to_string(),
16647                        launch_nonce: "self-nonce".to_string(),
16648                    }),
16649                    None,
16650                ),
16651            )
16652            .await
16653            .expect("self-route refusal returns a typed frame");
16654        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16655        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16656        assert!(
16657            self_rx.try_recv().is_err(),
16658            "self denial must not relay route.bind"
16659        );
16660    }
16661
16662    #[test]
16663    fn unsupported_channel_zero_frame_returns_error() {
16664        let handler = ControlHandler::default();
16665        let request = Frame::build(
16666            FrameType::Request,
16667            control_flags(),
16668            0,
16669            0,
16670            21,
16671            b"opaque".to_vec(),
16672        )
16673        .unwrap();
16674
16675        let response = handler
16676            .handle_control(ConnectionId::new(1), request)
16677            .unwrap();
16678
16679        assert_eq!(response[0].header.ty, FrameType::Error);
16680        assert_eq!(
16681            parse_error(&response[0])["code"],
16682            "unsupported_control_frame"
16683        );
16684    }
16685
16686    /// Blue/green swap at the control-plane boundary. The supervisor that opens
16687    /// a swap is not wired yet, so the candidate is registered here directly
16688    /// into the registry and forwarding candidate slots, the way the swap's
16689    /// HELLO admission will.
16690    mod swap {
16691        use super::*;
16692
16693        const INCUMBENT: ConnectionId = ConnectionId::new(30);
16694        const CANDIDATE: ConnectionId = ConnectionId::new(40);
16695
16696        struct Swap {
16697            registry: Arc<Registry>,
16698            forwarding: Arc<ForwardingTable>,
16699            handler: ControlHandler,
16700            incumbent_ctx: RouteCtx,
16701            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16702            candidate_ctx: RouteCtx,
16703            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16704        }
16705
16706        async fn swap_with_incumbent() -> Swap {
16707            let registry = Arc::new(Registry::default());
16708            let forwarding = Arc::new(ForwardingTable::default());
16709            let handler =
16710                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
16711            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16712            hello_via_sink(
16713                &handler,
16714                &incumbent_ctx,
16715                &mut incumbent_rx,
16716                hello_frame("aft", PROTOCOL_VERSION, 7),
16717            )
16718            .await;
16719            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16720            Swap {
16721                registry,
16722                forwarding,
16723                handler,
16724                incumbent_ctx,
16725                incumbent_rx,
16726                candidate_ctx,
16727                candidate_rx,
16728            }
16729        }
16730
16731        fn register_candidate(swap: &Swap, ready: Option<bool>) {
16732            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16733            candidate_manifest.ready = ready;
16734            let registration = swap
16735                .registry
16736                .register_candidate_with_control_ops(
16737                    candidate_manifest,
16738                    PROTOCOL_VERSION,
16739                    CANDIDATE,
16740                    module_baseline_control_ops(),
16741                )
16742                .unwrap();
16743            swap.forwarding
16744                .register_candidate_module_connection(
16745                    CANDIDATE,
16746                    "aft".to_string(),
16747                    PROTOCOL_VERSION,
16748                    manifest_concurrency(&registration.manifest),
16749                    swap.candidate_ctx.egress.clone(),
16750                )
16751                .unwrap();
16752        }
16753
16754        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16755            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16756            swap.registry.promote_candidate("aft").unwrap().unwrap();
16757            cutover.incumbent.unwrap()
16758        }
16759
16760        /// A swap candidate's HELLO reserves its candidate registry entry
16761        /// before installing its candidate forwarding endpoint. The swap
16762        /// supervisor promotes a ready candidate as soon as it reads it as
16763        /// registered, and promotion cuts forwarding over to the candidate
16764        /// endpoint, so the lookup the supervisor reads
16765        /// (`Registry::published_registration`) must not return the candidate
16766        /// until that endpoint exists. Otherwise the cutover finds no candidate
16767        /// endpoint and the swap fails as if the candidate had disconnected.
16768        #[tokio::test]
16769        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16770            let mut swap = swap_with_incumbent().await;
16771            let hold = Arc::new(std::sync::Barrier::new(2));
16772            let handler = swap
16773                .handler
16774                .clone()
16775                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16776            handler
16777                .supervisor
16778                .open_swap("aft", "candidate-nonce".to_string());
16779            let hello = {
16780                let handler = handler.clone();
16781                let candidate_ctx = swap.candidate_ctx.clone();
16782                std::thread::spawn(move || {
16783                    handler
16784                        .handle_hello(
16785                            candidate_ctx.connection_id,
16786                            Some(candidate_ctx.egress.clone()),
16787                            hello_frame_with_nonce(
16788                                "aft",
16789                                PROTOCOL_VERSION,
16790                                8,
16791                                Some("candidate-nonce"),
16792                            ),
16793                        )
16794                        .unwrap()
16795                })
16796            };
16797
16798            hold.wait();
16799            // Parked: the candidate slot is claimed but not published, and
16800            // forwarding has no candidate endpoint.
16801            assert!(swap
16802                .registry
16803                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16804                .unwrap()
16805                .is_some());
16806            assert!(swap
16807                .registry
16808                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16809                .unwrap()
16810                .is_none());
16811            assert!(swap
16812                .forwarding
16813                .module_endpoint_for_connection(CANDIDATE)
16814                .unwrap()
16815                .is_none());
16816
16817            hold.wait();
16818            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16819            assert_eq!(
16820                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16821                FrameType::HelloAck
16822            );
16823            let published = swap
16824                .registry
16825                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16826                .unwrap()
16827                .expect("candidate is published once its HELLO completes");
16828            assert_eq!(published.connection_id, CANDIDATE);
16829            assert!(swap
16830                .forwarding
16831                .module_endpoint_for_connection(CANDIDATE)
16832                .unwrap()
16833                .is_some());
16834        }
16835
16836        fn keyed_total(counters: &Value, key: &str) -> u64 {
16837            counters[key]
16838                .as_object()
16839                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16840                .unwrap_or(0)
16841        }
16842
16843        #[tokio::test]
16844        async fn replacement_logs_old_end_and_new_admission_once() {
16845            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16846            let mut swap = swap_with_incumbent().await;
16847            swap.handler
16848                .supervisor
16849                .open_swap("aft", "candidate-nonce".to_string());
16850            hello_via_sink(
16851                &swap.handler,
16852                &swap.candidate_ctx,
16853                &mut swap.candidate_rx,
16854                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16855            )
16856            .await;
16857            cutover(&swap);
16858            swap.handler.cleanup_connection(INCUMBENT).unwrap();
16859
16860            let captured = crate::router::test_log::captured_logs(&logs);
16861            println!("captured replacement registry logs:\n{captured}");
16862            let new_admissions: Vec<_> = captured
16863                .lines()
16864                .filter(|line| {
16865                    line.contains("connection_id=40")
16866                        && line.contains("swap candidate registered; not routable until cutover")
16867                })
16868                .collect();
16869            assert_eq!(
16870                new_admissions.len(),
16871                1,
16872                "unexpected admission logs: {captured}"
16873            );
16874            assert!(new_admissions[0].contains("module_id=aft"));
16875            assert!(new_admissions[0].contains("connection_id=40"));
16876            assert!(
16877                new_admissions[0].contains("swap candidate registered; not routable until cutover")
16878            );
16879            assert!(new_admissions[0].contains("ready=true"));
16880
16881            let old_admissions: Vec<_> = captured
16882                .lines()
16883                .filter(|line| {
16884                    line.contains("module registered module_id=aft ")
16885                        && line.contains("connection_id=30")
16886                })
16887                .collect();
16888            assert_eq!(
16889                old_admissions.len(),
16890                1,
16891                "unexpected incumbent admission: {captured}"
16892            );
16893
16894            let old_ends: Vec<_> = captured
16895                .lines()
16896                .filter(|line| {
16897                    line.contains("connection_id=30") && line.contains("module registration ended")
16898                })
16899                .collect();
16900            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16901            assert!(old_ends[0].contains("module_id=aft"));
16902            assert!(old_ends[0].contains("connection_id=30"));
16903            assert!(old_ends[0].contains("reason=replaced"));
16904            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16905            assert_eq!(
16906                captured
16907                    .lines()
16908                    .filter(|line| line.contains("module registration promoted"))
16909                    .count(),
16910                1,
16911                "unexpected promotion logs: {captured}"
16912            );
16913        }
16914
16915        /// An ack from the incumbent for a bind it was sent before cutover,
16916        /// arriving before the incumbent is drained. The incumbent is the live
16917        /// connection carrying every other client's routes, so the ack must
16918        /// not end it: the waiting client is told to retry, the reservation is
16919        /// given back, and the incumbent is told to drop just that binding.
16920        #[tokio::test]
16921        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16922            let mut swap = swap_with_incumbent().await;
16923            let handler = swap.handler.clone();
16924
16925            // A co-tenant route, bound on the incumbent before the swap.
16926            let cotenant = ConnectionId::new(31);
16927            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16928            let (cotenant_task, cotenant_bind) = relay_route_open(
16929                &handler,
16930                cotenant,
16931                &cotenant_ctx.egress,
16932                &mut swap.incumbent_rx,
16933                100,
16934                "aft",
16935                "swap-cotenant",
16936            )
16937            .await;
16938            handler
16939                .handle_control_frame(
16940                    &swap.incumbent_ctx,
16941                    route_bind_ack(cotenant_bind.header.corr),
16942                )
16943                .await
16944                .unwrap();
16945            assert!(cotenant_task.await.unwrap().is_empty());
16946            let (cotenant_channel, cotenant_epoch) =
16947                published_route(&cotenant_rx.recv().await.unwrap());
16948
16949            // A second route.open, relayed to the incumbent and not yet acked.
16950            let caller = ConnectionId::new(32);
16951            let (caller_ctx, mut caller_rx) = route_ctx(caller);
16952            let (caller_task, caller_bind) = relay_route_open(
16953                &handler,
16954                caller,
16955                &caller_ctx.egress,
16956                &mut swap.incumbent_rx,
16957                101,
16958                "aft",
16959                "swap-caller",
16960            )
16961            .await;
16962            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16963
16964            register_candidate(&swap, None);
16965            cutover(&swap);
16966
16967            // The incumbent acks after promotion and before any drain.
16968            let ack = handler
16969                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16970                .await;
16971            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16972            if module_loop_error.is_some() {
16973                // What the connection loop does with an untranslated router
16974                // error: end the connection, releasing every route on it.
16975                handler.cleanup_connection(INCUMBENT).unwrap();
16976            }
16977
16978            // 1. The incumbent's other routes survive.
16979            assert!(
16980                cotenant_rx.try_recv().is_err(),
16981                "the co-tenant route on the incumbent was torn down by one late ack: \
16982                 {module_loop_error:?}"
16983            );
16984            assert!(matches!(
16985                swap.forwarding
16986                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16987                    .unwrap(),
16988                DataRoute::Client(DataRouteState::Bound(_))
16989            ));
16990            assert_eq!(module_loop_error, None);
16991            assert!(swap
16992                .registry
16993                .get_module_by_connection(INCUMBENT)
16994                .unwrap()
16995                .is_some());
16996
16997            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
16998            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16999                .await
17000                .expect("the incumbent is told to drop the abandoned binding")
17001                .unwrap()
17002                .frame;
17003            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17004            assert_eq!(goodbye.header.channel, abandoned_channel);
17005            assert_eq!(goodbye.header.epoch, abandoned_epoch);
17006            assert!(swap.incumbent_rx.try_recv().is_err());
17007
17008            // 3. The waiting client gets a retryable refusal and no route.
17009            let response = caller_task.await.unwrap();
17010            assert_eq!(response.len(), 1);
17011            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17012            assert!(caller_rx.try_recv().is_err());
17013
17014            // 4. The reservation pair is given back, and the pending bind
17015            //    settled exactly once: one accepted open (the co-tenant) and one
17016            //    refused open (the caller), nothing counted twice.
17017            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17018            let counters = handler.counters().snapshot();
17019            assert_eq!(
17020                keyed_total(&counters, "route_open_accepted_by_principal"),
17021                1
17022            );
17023            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17024            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17025        }
17026
17027        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
17028        /// id would resolve to the promoted candidate and every new route.open
17029        /// would be refused as reloading, leaving neither process routable.
17030        #[tokio::test]
17031        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17032            let mut swap = swap_with_incumbent().await;
17033            register_candidate(&swap, None);
17034            let incumbent = cutover(&swap);
17035            swap.forwarding
17036                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17037                .unwrap()
17038                .expect("the incumbent is still registered");
17039
17040            let client = ConnectionId::new(33);
17041            let (client_ctx, mut client_rx) = route_ctx(client);
17042            let route_handler = swap.handler.clone();
17043            let open_ctx = RouteCtx {
17044                connection_id: client,
17045                egress: client_ctx.egress.clone(),
17046            };
17047            let mut route_task = tokio::spawn(async move {
17048                route_handler
17049                    .handle_control_frame(
17050                        &open_ctx,
17051                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17052                    )
17053                    .await
17054                    .unwrap()
17055            });
17056            let bind = tokio::select! {
17057                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17058                response = &mut route_task => {
17059                    let response = response.unwrap();
17060                    panic!(
17061                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
17062                        parse_error(&response[0])["code"]
17063                    );
17064                }
17065            };
17066            swap.handler
17067                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17068                .await
17069                .unwrap();
17070            assert!(route_task.await.unwrap().is_empty());
17071            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17072            match swap
17073                .forwarding
17074                .lookup_data_route(client, channel, epoch)
17075                .unwrap()
17076            {
17077                DataRoute::Client(DataRouteState::Bound(route)) => {
17078                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17079                }
17080                other => panic!("expected a bound route on the candidate, got {other:?}"),
17081            }
17082            assert!(swap.incumbent_rx.try_recv().is_err());
17083        }
17084
17085        /// A candidate declares itself ready with `catalog.update` on its own
17086        /// connection. If the connection-keyed registry lookups searched only the
17087        /// active slot, this would answer `not_registered` and the candidate
17088        /// would never become ready.
17089        #[tokio::test]
17090        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17091            let swap = swap_with_incumbent().await;
17092            register_candidate(&swap, Some(false));
17093            let update = Frame::build(
17094                FrameType::Request,
17095                control_flags(),
17096                0,
17097                0,
17098                55,
17099                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17100                    provides: manifest("aft", PROTOCOL_VERSION).provides,
17101                    capabilities: None,
17102                    ready: Some(true),
17103                })
17104                .unwrap(),
17105            )
17106            .unwrap();
17107
17108            let replies = swap
17109                .handler
17110                .handle_control_frame(&swap.candidate_ctx, update)
17111                .await
17112                .unwrap();
17113
17114            assert_eq!(replies.len(), 1);
17115            assert_eq!(
17116                replies[0].header.ty,
17117                FrameType::Response,
17118                "candidate catalog.update was refused: {:?}",
17119                serde_json::from_slice::<Value>(&replies[0].body).ok()
17120            );
17121            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17122            assert_eq!(
17123                swap.registry
17124                    .get_module("aft")
17125                    .unwrap()
17126                    .unwrap()
17127                    .connection_id,
17128                INCUMBENT
17129            );
17130        }
17131    }
17132
17133    /// The HELLO gate while the supervisor has a swap open: only the nonce it
17134    /// minted for the candidate admits a second process, into the candidate
17135    /// slot, and that check runs ahead of the reserved-module gate.
17136    mod swap_admission {
17137        use super::*;
17138
17139        const INCUMBENT_NONCE: &str = "incumbent-nonce";
17140        const CANDIDATE_NONCE: &str = "candidate-nonce";
17141
17142        fn handler_with_incumbent(
17143            module_id: &str,
17144            reserved: bool,
17145        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17146            let registry = Arc::new(Registry::default());
17147            let supervisor = SupervisorHandle::new();
17148            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17149            if reserved {
17150                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17151            }
17152            let handler =
17153                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
17154            let incumbent = handler
17155                .handle_control(
17156                    ConnectionId::new(1),
17157                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17158                )
17159                .unwrap();
17160            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17161            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17162            (registry, supervisor, handler)
17163        }
17164
17165        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
17166        /// admits every nonce, so while a swap is open the swap gate is the only
17167        /// thing between a key-holder and the candidate slot. A nonce the
17168        /// supervisor did not mint, or none at all, is refused, and neither the
17169        /// incumbent's registration nor the candidate slot moves.
17170        #[test]
17171        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17172            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17173
17174            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17175                let replies = handler
17176                    .handle_control(
17177                        ConnectionId::new(connection),
17178                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17179                    )
17180                    .unwrap();
17181                assert_eq!(replies[0].header.ty, FrameType::Error);
17182                assert_eq!(
17183                    parse_error(&replies[0])["code"],
17184                    "swap_token_invalid",
17185                    "nonce {nonce:?}"
17186                );
17187            }
17188            assert!(registry.get_candidate("aft").unwrap().is_none());
17189            assert_eq!(
17190                registry.get_module("aft").unwrap().unwrap().connection_id,
17191                ConnectionId::new(1)
17192            );
17193
17194            // Control: the minted token is admitted, into the candidate slot,
17195            // and only once.
17196            let admitted = handler
17197                .handle_control(
17198                    ConnectionId::new(4),
17199                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17200                )
17201                .unwrap();
17202            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17203            assert_eq!(
17204                registry
17205                    .get_candidate("aft")
17206                    .unwrap()
17207                    .unwrap()
17208                    .connection_id,
17209                ConnectionId::new(4)
17210            );
17211            assert_eq!(
17212                registry.get_module("aft").unwrap().unwrap().connection_id,
17213                ConnectionId::new(1),
17214                "the candidate must not take the active slot"
17215            );
17216            let replayed = handler
17217                .handle_control(
17218                    ConnectionId::new(5),
17219                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17220                )
17221                .unwrap();
17222            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17223
17224            // The case only this gate covers: the incumbent has died mid-swap,
17225            // so its duplicate refusal is gone too, and without the gate a
17226            // key-holder would take the id's ACTIVE slot.
17227            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17228            let squatter = handler
17229                .handle_control(
17230                    ConnectionId::new(6),
17231                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17232                )
17233                .unwrap();
17234            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17235            assert!(
17236                registry.get_module("aft").unwrap().is_none(),
17237                "a squatter took the active slot of an id being swapped"
17238            );
17239        }
17240
17241        /// Design mutation arm (iii). A reserved module's candidate presents a
17242        /// nonce the reserved gate has never seen (that gate holds the
17243        /// incumbent's), so the swap gate must run first or the candidate is
17244        /// refused `reserved_module` and a reserved module can never be swapped.
17245        #[test]
17246        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17247            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17248
17249            let replies = handler
17250                .handle_control(
17251                    ConnectionId::new(2),
17252                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17253                )
17254                .unwrap();
17255
17256            assert_eq!(
17257                replies[0].header.ty,
17258                FrameType::HelloAck,
17259                "reserved candidate refused: {:?}",
17260                serde_json::from_slice::<Value>(&replies[0].body).ok()
17261            );
17262            assert_eq!(
17263                registry
17264                    .get_candidate("vault")
17265                    .unwrap()
17266                    .unwrap()
17267                    .connection_id,
17268                ConnectionId::new(2)
17269            );
17270        }
17271
17272        /// With no swap open the gate is inert: the incumbent's reserved gate
17273        /// and duplicate refusal behave exactly as before.
17274        #[test]
17275        fn without_an_open_swap_the_ordinary_gates_decide() {
17276            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17277            supervisor.close_swap("vault");
17278
17279            let candidate = handler
17280                .handle_control(
17281                    ConnectionId::new(2),
17282                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17283                )
17284                .unwrap();
17285            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17286            let duplicate = handler
17287                .handle_control(
17288                    ConnectionId::new(3),
17289                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17290                )
17291                .unwrap();
17292            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17293            assert!(registry.get_candidate("vault").unwrap().is_none());
17294        }
17295    }
17296
17297    /// `scope.sync` and `scope.describe` through the real control handler: who
17298    /// may sync is decided by the registration and launch nonce of the module
17299    /// connection, never by the request body.
17300    mod scopes {
17301        mod idle_tests {
17302            use super::*;
17303            include!("scope_idle_tests.rs");
17304        }
17305        mod route_open_timings {
17306            use super::*;
17307            include!("route_open_timing_tests.rs");
17308        }
17309        mod route_open_refusal_log {
17310            use super::*;
17311            include!("route_open_refusal_log_tests.rs");
17312        }
17313        mod activity {
17314            include!("activity_scope_tests.rs");
17315        }
17316        mod delta {
17317            include!("scope_handler_delta_tests.rs");
17318        }
17319        mod call_modes {
17320            include!("scope_call_mode_tests.rs");
17321        }
17322        #[tokio::test]
17323        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17324            let rig = rig().await;
17325            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17326                "s".into(),
17327                1,
17328                "tool".into(),
17329            ))
17330            .unwrap();
17331            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17332            let (direct, _rx) = wide_ctx(999);
17333            for (ctx, expected) in [
17334                (&rig.owner, FrameType::Response),
17335                (&direct, FrameType::Error),
17336            ] {
17337                let frame =
17338                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17339                        .unwrap();
17340                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17341                assert_eq!(replies[0].header.ty, expected);
17342                if expected == FrameType::Response {
17343                    let reply: subc_control::ScopeServedByReply =
17344                        serde_json::from_slice(&replies[0].body).unwrap();
17345                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17346                } else {
17347                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17348                }
17349            }
17350            let reply = call(
17351                &rig.handler,
17352                &rig.owner,
17353                &ModuleControlRequestFromModule::ScopeDescribe {
17354                    owner: Principal::Reserved {
17355                        module_id: OWNER.into(),
17356                    },
17357                    scope_ref: "s".into(),
17358                },
17359            )
17360            .await;
17361            assert!(matches!(
17362                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17363                ModuleControlResponseToModule::ScopeDescribe { .. }
17364            ));
17365        }
17366
17367        use subc_protocol::scope::{
17368            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17369            ScopeStatus,
17370        };
17371
17372        use super::*;
17373
17374        const OWNER: &str = "prefrontal-core";
17375
17376        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17377            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17378        }
17379
17380        async fn call(
17381            handler: &ControlHandler,
17382            ctx: &RouteCtx,
17383            request: &ModuleControlRequestFromModule,
17384        ) -> Frame {
17385            let body = serde_json::to_vec(request).unwrap();
17386            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17387            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17388            assert_eq!(replies.len(), 1, "{replies:?}");
17389            replies.pop().unwrap()
17390        }
17391
17392        async fn sync(
17393            handler: &ControlHandler,
17394            ctx: &RouteCtx,
17395            generation: u64,
17396            scopes: Vec<ScopeRecord>,
17397        ) -> Result<ModuleControlResponseToModule, String> {
17398            let reply = call(
17399                handler,
17400                ctx,
17401                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17402            )
17403            .await;
17404            match reply.header.ty {
17405                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17406                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17407            }
17408        }
17409
17410        async fn describe(
17411            handler: &ControlHandler,
17412            ctx: &RouteCtx,
17413            owner: &str,
17414            scope_ref: &str,
17415        ) -> ModuleControlResponseToModule {
17416            let reply = call(
17417                handler,
17418                ctx,
17419                &ModuleControlRequestFromModule::ScopeDescribe {
17420                    owner: Principal::Reserved {
17421                        module_id: owner.to_string(),
17422                    },
17423                    scope_ref: scope_ref.to_string(),
17424                },
17425            )
17426            .await;
17427            assert_eq!(
17428                reply.header.ty,
17429                FrameType::Response,
17430                "{:?}",
17431                parse_error(&reply)
17432            );
17433            serde_json::from_slice(&reply.body).unwrap()
17434        }
17435
17436        #[tokio::test]
17437        async fn scope_describe_never_echoes_bind_proof_or_hash() {
17438            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17439            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17440            let mut rig = rig().await;
17441            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17442                .await;
17443            let _bound = rig
17444                .bound(
17445                    Some(AFT),
17446                    PLEXUS,
17447                    Some(
17448                        rig.selector("s", Some(1))
17449                            .with_bind_proof(Some(PROOF.into())),
17450                    ),
17451                )
17452                .await;
17453            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17454            let output = serde_json::to_string(&reply).unwrap();
17455            assert!(output.contains("\"status\":\"live\""));
17456            assert!(!output.contains(PROOF));
17457            assert!(!output.contains(HASH));
17458        }
17459
17460        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
17461        async fn module(
17462            handler: &ControlHandler,
17463            connection: u64,
17464            module_id: &str,
17465            nonce: Option<&str>,
17466        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17467            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17468            hello_via_sink(
17469                handler,
17470                &ctx,
17471                &mut rx,
17472                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17473            )
17474            .await;
17475            (ctx, rx)
17476        }
17477
17478        /// `direct` and every other client connection has no registration, so
17479        /// it can neither sync nor own a scope.
17480        #[tokio::test]
17481        async fn a_client_connection_cannot_sync_or_describe() {
17482            let handler = ControlHandler::new(Arc::new(Registry::default()));
17483            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17484            for request in [
17485                ModuleControlRequestFromModule::ScopeSync {
17486                    generation: 1,
17487                    scopes: vec![head("s", 1)],
17488                },
17489                ModuleControlRequestFromModule::ScopeDescribe {
17490                    owner: Principal::Direct,
17491                    scope_ref: "s".to_string(),
17492                },
17493            ] {
17494                let reply = call(&handler, &ctx, &request).await;
17495                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17496            }
17497            assert!(
17498                !handler
17499                    .scopes
17500                    .read()
17501                    .unwrap()
17502                    .describe(
17503                        &Principal::Reserved {
17504                            module_id: OWNER.to_string()
17505                        },
17506                        "s"
17507                    )
17508                    .owner_synced
17509            );
17510        }
17511
17512        /// A module the supervisor did not spawn registers without a launch
17513        /// nonce, so it is never an owner's current launch.
17514        #[tokio::test]
17515        async fn a_module_without_a_supervised_launch_cannot_sync() {
17516            let handler = ControlHandler::new(Arc::new(Registry::default()));
17517            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17518            assert_eq!(
17519                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17520                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17521            );
17522        }
17523
17524        #[tokio::test]
17525        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17526            let supervisor = SupervisorHandle::new();
17527            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17528            let handler = ControlHandler::new(Arc::new(Registry::default()))
17529                .with_supervisor(supervisor.clone());
17530            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17531            sync(&handler, &incumbent, 1, vec![head("s", 1)])
17532                .await
17533                .expect("the current launch syncs");
17534
17535            // A swap candidate registers with the swap token and is refused
17536            // while the incumbent keeps syncing.
17537            supervisor.open_swap(OWNER, "n2".to_string());
17538            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17539            assert_eq!(
17540                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17541                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17542            );
17543            sync(&handler, &incumbent, 2, vec![head("s", 1)])
17544                .await
17545                .expect("the serving owner syncs during the swap");
17546
17547            // The swap fails and is rolled back. The candidate never held sync
17548            // authority, and still cannot sync.
17549            supervisor.close_swap(OWNER);
17550            assert_eq!(
17551                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17552                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17553            );
17554            sync(&handler, &incumbent, 3, vec![head("s", 1)])
17555                .await
17556                .expect("the serving owner syncs after the rollback");
17557            handler.cleanup_connection(candidate.connection_id).unwrap();
17558
17559            // A swap that cuts over. Promotion records the candidate's nonce as
17560            // the module's spawn nonce, which is what `set_spawn_nonce` does
17561            // here; the promoted connection then takes authority at any
17562            // generation and the superseded incumbent is refused.
17563            supervisor.open_swap(OWNER, "n3".to_string());
17564            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17565            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17566            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17567                .await
17568                .expect("the promoted launch takes authority");
17569            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17570                panic!("unexpected reply {reply:?}");
17571            };
17572            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17573            assert_eq!(
17574                sync(&handler, &incumbent, 4, Vec::new()).await,
17575                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17576            );
17577        }
17578
17579        /// Authority dies with its connection: the cleanup path releases it,
17580        /// so the owner's next connection takes it at any generation.
17581        #[tokio::test]
17582        async fn closing_the_authority_connection_frees_sync_authority() {
17583            let supervisor = SupervisorHandle::new();
17584            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17585            let handler = ControlHandler::new(Arc::new(Registry::default()))
17586                .with_supervisor(supervisor.clone());
17587            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17588            sync(&handler, &first, 10, vec![head("s", 1)])
17589                .await
17590                .unwrap();
17591            handler.cleanup_connection(first.connection_id).unwrap();
17592
17593            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17594            sync(&handler, &second, 1, vec![head("s", 1)])
17595                .await
17596                .expect("the next connection takes the released authority");
17597        }
17598
17599        #[tokio::test]
17600        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17601            let supervisor = SupervisorHandle::new();
17602            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17603            let handler =
17604                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17605            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17606            sync(&handler, &first, 10, vec![head("s", 1)])
17607                .await
17608                .unwrap();
17609            handler
17610                .handle_control_frame(
17611                    &first,
17612                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17613                )
17614                .await
17615                .unwrap();
17616            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17617            sync(&handler, &second, 1, vec![head("s", 1)])
17618                .await
17619                .expect("GOODBYE releases authority even if the old socket remains open");
17620        }
17621
17622        #[tokio::test]
17623        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17624            let registry = Arc::new(Registry::default());
17625            let supervisor_handle = SupervisorHandle::new();
17626            let supervisor =
17627                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17628                    .with_handle(supervisor_handle.clone())
17629                    .with_daemon_incarnation("incarnation-7".to_string());
17630            // Configured with enabled: false, so the supervisor lists the
17631            // module without spawning a process for it.
17632            supervisor
17633                .supervise_configured(
17634                    ModuleSpec {
17635                        carry_containment: false,
17636                        module_id: OWNER.to_string(),
17637                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17638                        args: Vec::new(),
17639                        env: Vec::new(),
17640                        reserved: false,
17641                        reserved_prefixes: Vec::new(),
17642                        protocol: ModuleProtocol::Subc,
17643                        overlap: Default::default(),
17644                    },
17645                    false,
17646                )
17647                .unwrap();
17648            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17649            let handler =
17650                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17651            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17652
17653            // Configured but not yet synced: a reader waits for the owner.
17654            let ModuleControlResponseToModule::ScopeDescribe {
17655                status,
17656                daemon_incarnation,
17657                owner_synced,
17658                owner_configured,
17659                scope,
17660                ..
17661            } = describe(&handler, &reader, OWNER, "s").await
17662            else {
17663                panic!("not a describe reply");
17664            };
17665            assert_eq!(status, ScopeStatus::NotLive);
17666            assert_eq!(daemon_incarnation, "incarnation-7");
17667            assert!(!owner_synced);
17668            assert!(owner_configured);
17669            assert!(scope.is_none());
17670
17671            // Not a supervised module: the owner will never sync, and a reader
17672            // refuses rather than waits.
17673            let ModuleControlResponseToModule::ScopeDescribe {
17674                status,
17675                owner_configured,
17676                ..
17677            } = describe(&handler, &reader, "ghost", "s").await
17678            else {
17679                panic!("not a describe reply");
17680            };
17681            assert_eq!(status, ScopeStatus::NotLive);
17682            assert!(!owner_configured);
17683
17684            // Live, with the stamp fields and the computed owner_authorized.
17685            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17686            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17687            let ModuleControlResponseToModule::ScopeDescribe {
17688                status,
17689                scope_epoch,
17690                owner_synced,
17691                scope,
17692                ..
17693            } = describe(&handler, &reader, OWNER, "s").await
17694            else {
17695                panic!("not a describe reply");
17696            };
17697            assert_eq!(status, ScopeStatus::Live);
17698            assert_eq!(scope_epoch, Some(4));
17699            assert!(owner_synced);
17700            let stamp = scope.expect("a live scope carries its stamp");
17701            assert!(
17702                stamp.owner_authorized,
17703                "prefrontal-core is the default authority"
17704            );
17705            assert_eq!(stamp.kind, ScopeKind::Head);
17706        }
17707
17708        #[tokio::test]
17709        async fn scope_authority_owners_decides_owner_authorized() {
17710            let supervisor = SupervisorHandle::new();
17711            supervisor.set_spawn_nonce("broca", "b1".to_string());
17712            let handler = ControlHandler::new(Arc::new(Registry::default()))
17713                .with_supervisor(supervisor)
17714                .with_scope_authority_owners(vec!["broca".to_string()]);
17715            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17716            let mut gated = head("s", 1);
17717            gated.attributes.agent_id = Some("agent".to_string());
17718            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17719            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17720                describe(&handler, &broca, "broca", "s").await
17721            else {
17722                panic!("not a describe reply");
17723            };
17724            assert!(scope.unwrap().owner_authorized);
17725        }
17726
17727        /// With route admission, the stamp, the commit re-check and drains in
17728        /// place, the feature is advertised: the module ops in HELLO_ACK, and
17729        /// `scopes/v1` in HELLO_ACK and `server.describe`.
17730        #[tokio::test]
17731        async fn scope_ops_and_the_scopes_capability_are_advertised() {
17732            let handler = ControlHandler::new(Arc::new(Registry::default()));
17733            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17734            let ack = hello_via_sink(
17735                &handler,
17736                &ctx,
17737                &mut rx,
17738                hello_frame("m", PROTOCOL_VERSION, 1),
17739            )
17740            .await;
17741            let ack = parse_ack(&ack);
17742            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17743                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17744            }
17745            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17746
17747            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17748            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17749            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17750            let reply = handler
17751                .handle_control_frame(&client, frame)
17752                .await
17753                .unwrap()
17754                .pop()
17755                .unwrap();
17756            let ClientControlResponse::ServerDescribe { capabilities, .. } =
17757                serde_json::from_slice(&reply.body).unwrap()
17758            else {
17759                panic!("not a server.describe reply");
17760            };
17761            assert!(
17762                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17763                "{capabilities:?}"
17764            );
17765        }
17766
17767        // ---- route admission, stamps, commit re-check and drains ----------
17768
17769        const PLEXUS: &str = "plexus";
17770        const OTHER: &str = "other";
17771        const AFT: &str = "aft";
17772        const BROCA: &str = "broca";
17773        const MAGIC: &str = "magic-context";
17774
17775        fn nonce(module_id: &str) -> String {
17776            format!("nonce-{module_id}")
17777        }
17778
17779        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17780            let (tx, rx) = mpsc::channel(64);
17781            (
17782                RouteCtx {
17783                    connection_id: ConnectionId::new(connection),
17784                    egress: FrameSink::new(tx),
17785                },
17786                rx,
17787            )
17788        }
17789
17790        /// A daemon with a configured owner (prefrontal-core) registered on its
17791        /// own module connection, two routable targets (plexus, other), and
17792        /// launch nonces minted for the modules that open routes as carriers.
17793        struct Rig {
17794            handler: ControlHandler,
17795            forwarding: Arc<ForwardingTable>,
17796            owner: RouteCtx,
17797            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17798            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17799            generation: u64,
17800            next_connection: u64,
17801            _supervisor: Supervisor,
17802        }
17803
17804        async fn rig() -> Rig {
17805            rig_with_flow_support(true).await
17806        }
17807
17808        async fn rig_with_flow_support(flow_support: bool) -> Rig {
17809            rig_with_scope_capabilities(flow_support, false).await
17810        }
17811
17812        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17813            let mut provides = Vec::new();
17814            if flow_support {
17815                provides.push("flow-scopes/v1");
17816            }
17817            if run_support {
17818                provides.push("agent-run-scopes/v1");
17819            }
17820            rig_with_target_capabilities(&provides).await
17821        }
17822
17823        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17824            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17825            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17826            // Decoder versions alone cannot admit scoped calls: only a capability
17827            // declaration promises the required run or flow behaviour.
17828            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17829            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17830            Frame::build(
17831                FrameType::Hello,
17832                control_flags(),
17833                0,
17834                0,
17835                connection,
17836                serde_json::to_vec(&body).unwrap(),
17837            )
17838            .unwrap()
17839        }
17840
17841        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17842            let registry = Arc::new(Registry::default());
17843            let forwarding = Arc::new(ForwardingTable::default());
17844            let supervisor_handle = SupervisorHandle::new();
17845            let supervisor =
17846                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17847                    .with_handle(supervisor_handle.clone());
17848            supervisor
17849                .supervise_configured(
17850                    ModuleSpec {
17851                        carry_containment: false,
17852                        module_id: OWNER.to_string(),
17853                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17854                        args: Vec::new(),
17855                        env: Vec::new(),
17856                        reserved: false,
17857                        reserved_prefixes: Vec::new(),
17858                        protocol: ModuleProtocol::Subc,
17859                        overlap: Default::default(),
17860                    },
17861                    false,
17862                )
17863                .unwrap();
17864            for module_id in [OWNER, AFT, BROCA, MAGIC] {
17865                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17866            }
17867            let handler =
17868                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17869                    .with_supervisor(supervisor_handle);
17870            let (owner, mut owner_rx) = wide_ctx(1);
17871            hello_via_sink(
17872                &handler,
17873                &owner,
17874                &mut owner_rx,
17875                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17876            )
17877            .await;
17878            let mut modules = BTreeMap::new();
17879            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17880                let (ctx, mut rx) = wide_ctx(connection);
17881                let hello = hello_with_capabilities(module_id, connection, provides);
17882                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17883                modules.insert(module_id.to_string(), (ctx, rx));
17884            }
17885            Rig {
17886                handler,
17887                forwarding,
17888                owner,
17889                _owner_rx: owner_rx,
17890                modules,
17891                generation: 0,
17892                next_connection: 100,
17893                _supervisor: supervisor,
17894            }
17895        }
17896
17897        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17898            ScopeCarrier::new(Principal::Reserved {
17899                module_id: module_id.to_string(),
17900            })
17901            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17902        }
17903
17904        /// The scope most tests open under: aft carries to any module, broca
17905        /// only to plexus and other, and the owner delegates as agent-1.
17906        fn session(scope_epoch: u64) -> ScopeRecord {
17907            let mut record = head("s", scope_epoch);
17908            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17909            record.attributes.agent_id = Some("agent-1".to_string());
17910            record.attributes.delegates = true;
17911            record
17912        }
17913
17914        impl Rig {
17915            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17916                self.generation += 1;
17917                sync(&self.handler, &self.owner, self.generation, scopes)
17918                    .await
17919                    .expect("the owner's sync is accepted");
17920            }
17921
17922            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17923                ScopeSelector::new(
17924                    Principal::Reserved {
17925                        module_id: OWNER.to_string(),
17926                    },
17927                    scope_ref.to_string(),
17928                    scope_epoch,
17929                )
17930            }
17931
17932            fn open_frame(
17933                &mut self,
17934                opener: Option<&str>,
17935                target: &str,
17936                scope: Option<ScopeSelector>,
17937            ) -> (
17938                RouteCtx,
17939                mpsc::Receiver<crate::router::OutboundFrame>,
17940                Frame,
17941            ) {
17942                self.next_connection += 1;
17943                let (ctx, rx) = wide_ctx(self.next_connection);
17944                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17945                    busy_refusals: None,
17946                    target: RouteTarget::ToolProvider {
17947                        module_id: target.to_string(),
17948                    },
17949                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
17950                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
17951                        module_id: module_id.to_string(),
17952                        launch_nonce: nonce(module_id),
17953                    }),
17954                    consumer_capabilities: None,
17955                    role_versions: None,
17956                    admission_facts: None,
17957                    scope,
17958                })
17959                .unwrap();
17960                let frame = Frame::build(
17961                    FrameType::Request,
17962                    control_flags(),
17963                    0,
17964                    0,
17965                    self.next_connection,
17966                    body,
17967                )
17968                .unwrap();
17969                (ctx, rx, frame)
17970            }
17971
17972            /// Open and expect a refusal before anything is relayed.
17973            async fn refused(
17974                &mut self,
17975                opener: Option<&str>,
17976                target: &str,
17977                scope: Option<ScopeSelector>,
17978            ) -> String {
17979                self.refusal_body(opener, target, scope).await["code"]
17980                    .as_str()
17981                    .unwrap()
17982                    .to_string()
17983            }
17984
17985            async fn refusal_body(
17986                &mut self,
17987                opener: Option<&str>,
17988                target: &str,
17989                scope: Option<ScopeSelector>,
17990            ) -> Value {
17991                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17992                let replies = tokio::time::timeout(
17993                    Duration::from_secs(2),
17994                    self.handler.handle_control_frame(&ctx, frame),
17995                )
17996                .await
17997                .expect("the open must be refused before waiting for a bind ack")
17998                .unwrap();
17999                assert_eq!(replies.len(), 1, "{replies:?}");
18000                assert_eq!(replies[0].header.ty, FrameType::Error);
18001                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18002                assert!(
18003                    module_rx.try_recv().is_err(),
18004                    "a refused open relays nothing"
18005                );
18006                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18007                parse_error(&replies[0])
18008            }
18009
18010            /// Start an open and return its task and the bind the target got.
18011            async fn relayed(
18012                &mut self,
18013                opener: Option<&str>,
18014                target: &str,
18015                scope: Option<ScopeSelector>,
18016            ) -> Relayed {
18017                self.relayed_with_legacy_root(opener, target, scope, None)
18018                    .await
18019            }
18020
18021            async fn relayed_with_legacy_root(
18022                &mut self,
18023                opener: Option<&str>,
18024                target: &str,
18025                scope: Option<ScopeSelector>,
18026                legacy_root: Option<&std::path::Path>,
18027            ) -> Relayed {
18028                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18029                if let Some(root) = legacy_root {
18030                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18031                    body["identity"]["project_root"] = serde_json::json!(root);
18032                    frame.body = serde_json::to_vec(&body).unwrap();
18033                }
18034                let handler = self.handler.clone();
18035                let task_ctx = ctx.clone();
18036                let task = tokio::spawn(async move {
18037                    handler
18038                        .handle_control_frame(&task_ctx, frame)
18039                        .await
18040                        .unwrap()
18041                });
18042                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18043                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18044                    .await
18045                    .expect("the target receives the relayed route.bind")
18046                    .unwrap()
18047                    .frame;
18048                Relayed {
18049                    target: target.to_string(),
18050                    client: ctx,
18051                    client_rx: rx,
18052                    task,
18053                    bind,
18054                }
18055            }
18056
18057            async fn ack(&self, relayed: &Relayed) {
18058                let (module, _) = &self.modules[&relayed.target];
18059                self.handler
18060                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18061                    .await
18062                    .unwrap();
18063            }
18064
18065            /// Open, ack and return the bound route.
18066            async fn bound(
18067                &mut self,
18068                opener: Option<&str>,
18069                target: &str,
18070                scope: Option<ScopeSelector>,
18071            ) -> Bound {
18072                let relayed = self.relayed(opener, target, scope).await;
18073                self.commit_relayed(relayed).await
18074            }
18075
18076            async fn bound_with_legacy_root(
18077                &mut self,
18078                opener: Option<&str>,
18079                target: &str,
18080                scope: Option<ScopeSelector>,
18081                root: &std::path::Path,
18082            ) -> Bound {
18083                let relayed = self
18084                    .relayed_with_legacy_root(opener, target, scope, Some(root))
18085                    .await;
18086                self.commit_relayed(relayed).await
18087            }
18088
18089            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18090                self.ack(&relayed).await;
18091                let Relayed {
18092                    target,
18093                    client,
18094                    mut client_rx,
18095                    task,
18096                    bind,
18097                } = relayed;
18098                assert!(
18099                    task.await.unwrap().is_empty(),
18100                    "the open is answered by commit"
18101                );
18102                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18103                Bound {
18104                    target,
18105                    client,
18106                    client_rx,
18107                    channel,
18108                    epoch,
18109                    bind,
18110                }
18111            }
18112
18113            fn live(&self, route: &Bound) -> bool {
18114                matches!(
18115                    self.forwarding
18116                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18117                        .unwrap(),
18118                    DataRoute::Client(DataRouteState::Bound(_))
18119                )
18120            }
18121        }
18122
18123        struct Relayed {
18124            target: String,
18125            client: RouteCtx,
18126            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18127            task: tokio::task::JoinHandle<Vec<Frame>>,
18128            bind: Frame,
18129        }
18130
18131        struct Bound {
18132            target: String,
18133            client: RouteCtx,
18134            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18135            channel: u16,
18136            epoch: u32,
18137            bind: Frame,
18138        }
18139
18140        impl Bound {
18141            /// The reason of the `route.closed` this client was sent, after
18142            /// checking it also got a GOODBYE on exactly this route.
18143            fn closed_reason(&mut self) -> RouteCloseReason {
18144                let mut reason = None;
18145                let mut goodbye = false;
18146                while let Ok(outbound) = self.client_rx.try_recv() {
18147                    let frame = outbound.frame;
18148                    match frame.header.ty {
18149                        FrameType::Goodbye => {
18150                            assert_eq!(
18151                                (frame.header.channel, frame.header.epoch),
18152                                (self.channel, self.epoch)
18153                            );
18154                            goodbye = true;
18155                        }
18156                        FrameType::Push => {
18157                            let ClientControlPush::RouteClosed {
18158                                reason: r,
18159                                module_id,
18160                                ..
18161                            } = serde_json::from_slice(&frame.body).unwrap()
18162                            else {
18163                                panic!("unexpected push");
18164                            };
18165                            assert_eq!(module_id, self.target);
18166                            reason = Some(r);
18167                        }
18168                        other => panic!("unexpected frame {other:?}"),
18169                    }
18170                }
18171                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18172                reason.expect("the client is told why the route closed")
18173            }
18174
18175            fn untouched(&mut self) -> bool {
18176                self.client_rx.try_recv().is_err()
18177            }
18178
18179            fn stamp(&self) -> Option<ScopeStamp> {
18180                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18181                    ModuleControlRequest::RouteBind { scope, .. } => scope,
18182                    other => panic!("expected a route.bind, got {other:?}"),
18183                }
18184            }
18185        }
18186
18187        #[tokio::test]
18188        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18189        ) {
18190            let mut rig = rig().await;
18191            let mut record = session(1);
18192            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18193            record.child_owners = vec![Principal::Reserved {
18194                module_id: MAGIC.to_string(),
18195            }];
18196            rig.sync(vec![record]).await;
18197            let scope = || Some(rig_selector("s", Some(1)));
18198
18199            // Admitted: the owner, a bare carrier to any module, a targeted
18200            // carrier to its listed module.
18201            rig.bound(Some(OWNER), PLEXUS, scope()).await;
18202            rig.bound(Some(AFT), OTHER, scope()).await;
18203            rig.bound(Some(BROCA), PLEXUS, scope()).await;
18204
18205            // Refused scope_not_carrier: a targeted carrier to an unlisted
18206            // module, a module that is not listed at all (a child owner is not
18207            // a carrier), and a direct key-holder.
18208            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18209                assert_eq!(
18210                    rig.refused(opener, target, scope()).await,
18211                    error_codes::SCOPE_NOT_CARRIER,
18212                    "{opener:?} -> {target}"
18213                );
18214            }
18215        }
18216
18217        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18218            ScopeSelector::new(
18219                Principal::Reserved {
18220                    module_id: OWNER.to_string(),
18221                },
18222                scope_ref.to_string(),
18223                scope_epoch,
18224            )
18225        }
18226
18227        #[tokio::test]
18228        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18229            let mut rig = rig().await;
18230            rig.sync(vec![session(1)]).await;
18231            for opener in [OWNER, AFT] {
18232                assert_eq!(
18233                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18234                        .await,
18235                    error_codes::SCOPE_EPOCH_REQUIRED,
18236                    "{opener}"
18237                );
18238            }
18239        }
18240
18241        #[tokio::test]
18242        async fn admission_separates_not_synced_not_live_and_ended() {
18243            let mut rig = rig().await;
18244            // Before the configured owner's first sync: retryable.
18245            let code = rig
18246                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18247                .await;
18248            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18249            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18250
18251            // An owner that is not configured will never sync: terminal.
18252            let ghost = ScopeSelector::new(
18253                Principal::Reserved {
18254                    module_id: "ghost".to_string(),
18255                },
18256                "s".to_string(),
18257                Some(1),
18258            );
18259            assert_eq!(
18260                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18261                error_codes::SCOPE_NOT_LIVE
18262            );
18263
18264            rig.sync(vec![session(2)]).await;
18265            assert_eq!(
18266                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18267                    .await,
18268                error_codes::SCOPE_NOT_LIVE
18269            );
18270            for epoch in [1, 3] {
18271                assert_eq!(
18272                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18273                        .await,
18274                    error_codes::SCOPE_ENDED,
18275                    "epoch {epoch}"
18276                );
18277            }
18278            // Control: the live epoch is admitted.
18279            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18280                .await;
18281        }
18282
18283        #[tokio::test]
18284        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18285            let mut rig = rig().await;
18286            rig.sync(vec![session(1)]).await;
18287            let route = rig
18288                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18289                .await;
18290            let stamp = route.stamp().expect("a scoped bind carries the stamp");
18291            assert_eq!(stamp.scope_ref, "s");
18292            assert_eq!(stamp.scope_epoch, 1);
18293            assert_eq!(stamp.kind, ScopeKind::Head);
18294            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18295            assert!(stamp.attributes.delegates);
18296            assert!(stamp.owner_authorized);
18297            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18298            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18299
18300            // broca owns a scope of its own on its own module connection; it is
18301            // not in scope_authority_owners, so its stamp is not authorized.
18302            let (broca, mut broca_rx) = wide_ctx(50);
18303            hello_via_sink(
18304                &rig.handler,
18305                &broca,
18306                &mut broca_rx,
18307                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18308            )
18309            .await;
18310            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18311                .await
18312                .unwrap();
18313            let own = ScopeSelector::new(
18314                Principal::Reserved {
18315                    module_id: BROCA.to_string(),
18316                },
18317                "b".to_string(),
18318                Some(1),
18319            );
18320            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18321            assert!(!route.stamp().unwrap().owner_authorized);
18322        }
18323
18324        #[tokio::test]
18325        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18326            let mut rig = rig().await;
18327            let mut record = head("s", 1);
18328            record.carriers = vec![carrier(AFT, None)];
18329            let flow_id = "Flow:run-7/step_2!~";
18330            record.attributes.flow_id = Some(flow_id.to_string());
18331            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18332                .await
18333                .unwrap();
18334            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18335                panic!("not a sync reply");
18336            };
18337            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18338            let route = rig
18339                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18340                .await;
18341            assert!(rig.live(&route), "the stamped bind committed");
18342            let stamp = route.stamp().expect("a flow scope carries a stamp");
18343            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18344            assert_eq!(stamp.attributes.agent_id, None);
18345            assert!(!stamp.attributes.delegates);
18346            assert!(stamp.owner_authorized);
18347            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18348            assert_eq!(unscoped.stamp(), None);
18349        }
18350
18351        #[tokio::test]
18352        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18353            let mut rig = rig_with_flow_support(false).await;
18354            let mut record = session(1);
18355            record.attributes.flow_id = Some("flow:7".to_string());
18356            rig.sync(vec![record]).await;
18357            for opener in [OWNER, AFT] {
18358                let body = rig
18359                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18360                    .await;
18361                assert_eq!(body["code"], "target_flow_unsupported");
18362                let message = body["message"].as_str().unwrap();
18363                for required in [PLEXUS, "flow-scopes/v1"] {
18364                    assert!(message.contains(required), "{message}");
18365                }
18366            }
18367        }
18368
18369        #[tokio::test]
18370        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18371            let mut rig = rig_with_flow_support(true).await;
18372            let mut record = session(1);
18373            record.attributes.flow_id = Some("flow:7".to_string());
18374            rig.sync(vec![record]).await;
18375            for opener in [OWNER, AFT] {
18376                let route = rig
18377                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18378                    .await;
18379                assert!(rig.live(&route));
18380                assert_eq!(
18381                    route.stamp().unwrap().attributes.flow_id.as_deref(),
18382                    Some("flow:7")
18383                );
18384            }
18385        }
18386
18387        #[tokio::test]
18388        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18389            use std::future::Future;
18390
18391            let mut rig = rig_with_flow_support(true).await;
18392            let mut record = session(1);
18393            record.attributes.flow_id = Some("flow:7".to_string());
18394            rig.sync(vec![record]).await;
18395            let (client, mut client_rx, frame) =
18396                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18397            // Hold the route-open response permit so admission sees the first
18398            // target but relay reservation cannot capture an endpoint yet.
18399            for _ in 0..64 {
18400                client.egress.try_send(route_bind_ack(1)).unwrap();
18401            }
18402            let handler = rig.handler.clone();
18403            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18404            std::future::poll_fn(|cx| {
18405                assert!(open.as_mut().poll(cx).is_pending());
18406                std::task::Poll::Ready(())
18407            })
18408            .await;
18409            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18410
18411            let old_connection = rig.modules[PLEXUS].0.connection_id;
18412            rig.handler.cleanup_connection(old_connection).unwrap();
18413            let (replacement, mut replacement_rx) = wide_ctx(200);
18414            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18415            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18416            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18417            let hello = Frame::build(
18418                FrameType::Hello,
18419                control_flags(),
18420                0,
18421                0,
18422                200,
18423                serde_json::to_vec(&body).unwrap(),
18424            )
18425            .unwrap();
18426            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18427
18428            client_rx.try_recv().unwrap();
18429            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18430                .await
18431                .expect("the replacement is refused without waiting for a bind ack")
18432                .unwrap();
18433            assert_eq!(replies.len(), 1);
18434            let body = parse_error(&replies[0]);
18435            assert_eq!(body["code"], "target_flow_unsupported");
18436            for required in [PLEXUS, "flow-scopes/v1"] {
18437                assert!(body["message"].as_str().unwrap().contains(required));
18438            }
18439            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18440            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18441            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18442            assert!(rig
18443                .handler
18444                .registry
18445                .get_module_by_connection(replacement.connection_id)
18446                .unwrap()
18447                .is_some());
18448        }
18449
18450        #[tokio::test]
18451        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18452            use std::future::Future;
18453            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18454            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18455            const NEXT_HASH: &str =
18456                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18457            let mut rig = rig().await;
18458            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18459            record.carriers.push(ScopeCarrier::new(Principal::Direct));
18460            rig.sync(vec![record.clone()]).await;
18461            let (client, mut rx, frame) = rig.open_frame(
18462                None,
18463                PLEXUS,
18464                Some(
18465                    rig.selector("s", Some(1))
18466                        .with_bind_proof(Some(PROOF.into())),
18467                ),
18468            );
18469            // Hold egress reservation while the scope owner changes its bind-proof
18470            // hash, so a proof valid at admission must be checked again at relay.
18471            for _ in 0..64 {
18472                client.egress.try_send(route_bind_ack(1)).unwrap();
18473            }
18474            let handler = rig.handler.clone();
18475            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18476            std::future::poll_fn(|cx| {
18477                assert!(open.as_mut().poll(cx).is_pending());
18478                std::task::Poll::Ready(())
18479            })
18480            .await;
18481            record.bind_proof_sha256 = Some(NEXT_HASH.into());
18482            rig.sync(vec![record]).await;
18483            rx.try_recv().unwrap();
18484            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18485                .await
18486                .unwrap()
18487                .unwrap();
18488            assert_eq!(
18489                parse_error(&replies[0])["code"],
18490                error_codes::SCOPE_BIND_PROOF_MISMATCH
18491            );
18492            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18493            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18494            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18495        }
18496
18497        #[tokio::test]
18498        async fn scope_bind_proof_reserved_openers_ignore_proof() {
18499            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18500            let mut rig = rig().await;
18501            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18502                .await;
18503            for opener in [OWNER, AFT] {
18504                for proof in [None, Some("malformed+proof=")] {
18505                    let route = rig
18506                        .bound(
18507                            Some(opener),
18508                            PLEXUS,
18509                            Some(
18510                                rig.selector("s", Some(1))
18511                                    .with_bind_proof(proof.map(str::to_string)),
18512                            ),
18513                        )
18514                        .await;
18515                    assert!(rig.live(&route));
18516                }
18517            }
18518        }
18519
18520        #[tokio::test]
18521        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18522            use std::future::Future;
18523            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18524            const NEXT_HASH: &str =
18525                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18526            let mut rig = rig().await;
18527            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18528                .await;
18529            let (client, mut rx, frame) =
18530                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18531            for _ in 0..64 {
18532                client.egress.try_send(route_bind_ack(1)).unwrap();
18533            }
18534            let handler = rig.handler.clone();
18535            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18536            std::future::poll_fn(|cx| {
18537                assert!(open.as_mut().poll(cx).is_pending());
18538                std::task::Poll::Ready(())
18539            })
18540            .await;
18541            rig.sync(vec![
18542                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18543            ])
18544            .await;
18545            rx.try_recv().unwrap();
18546            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18547            let bind = std::future::poll_fn(|cx| {
18548                assert!(open.as_mut().poll(cx).is_pending());
18549                module_rx.poll_recv(cx)
18550            })
18551            .await
18552            .unwrap()
18553            .frame;
18554            handler
18555                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18556                .await
18557                .unwrap();
18558            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18559                .await
18560                .unwrap()
18561                .unwrap();
18562            assert_eq!(replies.len(), 1);
18563            // Proof rechecking allowed the relay. Independent tag settlement
18564            // still asks the opener to retry after any same-epoch content change.
18565            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18566            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18567            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18568            let route = rig
18569                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18570                .await;
18571            assert!(rig.live(&route));
18572        }
18573
18574        #[tokio::test]
18575        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18576        {
18577            let mut rig = rig_with_flow_support(false).await;
18578            rig.sync(vec![session(1)]).await;
18579            let route = rig
18580                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18581                .await;
18582            assert!(rig.live(&route));
18583            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18584            let mut record = session(1);
18585            record.attributes.flow_id = Some("flow:7".to_string());
18586            rig.sync(vec![record]).await;
18587            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18588            assert!(rig.live(&unscoped));
18589            assert_eq!(unscoped.stamp(), None);
18590        }
18591
18592        #[tokio::test]
18593        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18594            let mut rig = rig().await;
18595            let mut record = session(1);
18596            record.attributes.flow_id = Some("flow:7".to_string());
18597            rig.sync(vec![record.clone()]).await;
18598            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18599            let mut owner_route = rig
18600                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18601                .await;
18602            let mut carrier_route = rig
18603                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18604                .await;
18605            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18606            rig.sync(vec![record.clone()]).await;
18607            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18608            assert!(rig.live(&owner_route) && owner_route.untouched());
18609            assert!(rig.live(&carrier_route) && carrier_route.untouched());
18610
18611            record.attributes.flow_id = Some("flow:8".to_string());
18612            rig.sync(vec![record]).await;
18613            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18614            let before = before.unwrap();
18615            assert_eq!(after.scope_epoch, before.scope_epoch);
18616            assert!(after.version > before.version);
18617            for route in [&mut owner_route, &mut carrier_route] {
18618                assert!(!rig.live(route));
18619                assert_eq!(
18620                    route.closed_reason(),
18621                    RouteCloseReason::ScopeDelegationChanged
18622                );
18623            }
18624            assert!(rig.live(&unscoped) && unscoped.untouched());
18625            // Each provider also receives a GOODBYE for its drained route;
18626            // consume it before expecting the next route.bind on that sink.
18627            for target in [PLEXUS, OTHER] {
18628                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18629                let goodbye = module_rx
18630                    .try_recv()
18631                    .expect("the provider sees the drain")
18632                    .frame;
18633                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18634                assert!(module_rx.try_recv().is_err());
18635            }
18636            let rebound = rig
18637                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18638                .await;
18639            assert_eq!(
18640                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18641                Some("flow:8")
18642            );
18643        }
18644
18645        /// The owner's sync lands between admission and the module's ack. The
18646        /// open is refused by name, the module's other routes stay up, and the
18647        /// reserved pair is released. Changed content is retryable; an ended
18648        /// scope is not.
18649        #[tokio::test]
18650        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18651            let mut rig = rig().await;
18652            rig.sync(vec![session(1)]).await;
18653            let mut cotenant = rig
18654                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18655                .await;
18656
18657            let mut changed = session(1);
18658            changed.child_owners.push(Principal::Reserved {
18659                module_id: MAGIC.to_string(),
18660            });
18661            let mut ended = None;
18662            for (code, next) in [
18663                (error_codes::SCOPE_CHANGED, vec![changed]),
18664                (error_codes::SCOPE_ENDED, Vec::new()),
18665            ] {
18666                let relayed = rig
18667                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18668                    .await;
18669                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18670                ended = Some(next.is_empty());
18671                rig.sync(next).await;
18672                rig.ack(&relayed).await;
18673                let replies = relayed.task.await.unwrap();
18674                assert_eq!(replies.len(), 1, "{replies:?}");
18675                assert_eq!(parse_error(&replies[0])["code"], code);
18676                assert_eq!(
18677                    subc_protocol::error_codes::is_retryable_route_open(code),
18678                    code == error_codes::SCOPE_CHANGED
18679                );
18680                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18681                // The module is told to drop just the binding it created.
18682                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18683                // Collected, because ending the scope also closes the co-tenant
18684                // route, whose GOODBYE comes first.
18685                let mut goodbyes = Vec::new();
18686                while let Ok(outbound) = plexus_rx.try_recv() {
18687                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18688                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18689                }
18690                assert!(
18691                    goodbyes.contains(&(bind_channel, bind_epoch)),
18692                    "{goodbyes:?}"
18693                );
18694                assert!(rig
18695                    .handler
18696                    .registry
18697                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18698                    .unwrap()
18699                    .is_some());
18700            }
18701            assert_eq!(ended, Some(true));
18702            // The co-tenant stayed up through the change, and closed only when
18703            // the scope ended, by the drain rule rather than by the commit.
18704            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18705        }
18706
18707        /// Each row of the drain table on one set of routes: the owner's, a
18708        /// bare carrier's, and a targeted carrier's to each of its targets.
18709        #[tokio::test]
18710        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18711            struct Case {
18712                name: &'static str,
18713                change: fn(&mut ScopeRecord),
18714                /// Closed routes by index: owner->plexus, aft->plexus,
18715                /// broca->plexus, broca->other.
18716                closed: [Option<RouteCloseReason>; 4],
18717            }
18718            use RouteCloseReason::*;
18719            let cases = [
18720                Case {
18721                    name: "a carrier entry removed",
18722                    change: |r| {
18723                        r.carriers.retain(|c| {
18724                            c.principal
18725                                != Principal::Reserved {
18726                                    module_id: AFT.to_string(),
18727                                }
18728                        })
18729                    },
18730                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18731                },
18732                Case {
18733                    name: "a target removed from a carrier",
18734                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18735                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
18736                },
18737                Case {
18738                    name: "a bare carrier narrowed to targets",
18739                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18740                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18741                },
18742                Case {
18743                    name: "delegates turned off",
18744                    change: |r| r.attributes.delegates = false,
18745                    closed: [Some(ScopeDelegationChanged); 4],
18746                },
18747                Case {
18748                    name: "agent_id changed",
18749                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18750                    closed: [Some(ScopeDelegationChanged); 4],
18751                },
18752                Case {
18753                    name: "a carrier added, child owners changed, the record re-sent",
18754                    change: |r| {
18755                        r.carriers.push(carrier(MAGIC, None));
18756                        r.child_owners.push(Principal::Reserved {
18757                            module_id: MAGIC.to_string(),
18758                        });
18759                    },
18760                    closed: [None; 4],
18761                },
18762                Case {
18763                    name: "a target added",
18764                    change: |r| {
18765                        r.carriers[1]
18766                            .targets
18767                            .as_mut()
18768                            .unwrap()
18769                            .push("third".to_string())
18770                    },
18771                    closed: [None; 4],
18772                },
18773                Case {
18774                    name: "delegates turned on",
18775                    change: |r| r.attributes.delegates = true,
18776                    closed: [None; 4],
18777                },
18778            ];
18779            for case in cases {
18780                let mut rig = rig().await;
18781                rig.sync(vec![session(1)]).await;
18782                let scope = || Some(rig_selector("s", Some(1)));
18783                let mut routes = [
18784                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
18785                    rig.bound(Some(AFT), PLEXUS, scope()).await,
18786                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
18787                    rig.bound(Some(BROCA), OTHER, scope()).await,
18788                ];
18789                let mut record = session(1);
18790                (case.change)(&mut record);
18791                rig.sync(vec![record]).await;
18792                for (index, expected) in case.closed.iter().enumerate() {
18793                    let route = &mut routes[index];
18794                    match expected {
18795                        Some(reason) => {
18796                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
18797                            assert_eq!(
18798                                route.closed_reason(),
18799                                *reason,
18800                                "{}: route {index}",
18801                                case.name
18802                            );
18803                        }
18804                        None => {
18805                            assert!(rig.live(route), "{}: route {index} closed", case.name);
18806                            assert!(
18807                                route.untouched(),
18808                                "{}: route {index} was told something",
18809                                case.name
18810                            );
18811                        }
18812                    }
18813                }
18814            }
18815        }
18816
18817        #[tokio::test]
18818        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18819            // Removed, and replaced by a higher epoch.
18820            for next in [Vec::new(), vec![session(2)]] {
18821                let mut rig = rig().await;
18822                rig.sync(vec![session(1)]).await;
18823                let mut route = rig
18824                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18825                    .await;
18826                rig.sync(next).await;
18827                assert!(!rig.live(&route));
18828                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18829            }
18830
18831            // A child whose parent ends: its routes close as parent-ended, the
18832            // child stays live, and routes under the parent close as ended.
18833            let mut rig = rig().await;
18834            let mut child = session(1);
18835            child.scope_ref = "child".to_string();
18836            child.kind = ScopeKind::Worker;
18837            child.parent = Some(ScopeParent::new(
18838                Principal::Reserved {
18839                    module_id: OWNER.to_string(),
18840                },
18841                "s".to_string(),
18842                1,
18843            ));
18844            rig.sync(vec![session(1), child.clone()]).await;
18845            let mut child_route = rig
18846                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18847                .await;
18848            assert_eq!(
18849                child_route.stamp().unwrap().parent_state,
18850                Some(ParentState::Linked)
18851            );
18852            rig.sync(vec![child]).await;
18853            assert!(!rig.live(&child_route));
18854            assert_eq!(
18855                child_route.closed_reason(),
18856                RouteCloseReason::ScopeParentEnded
18857            );
18858        }
18859
18860        #[tokio::test]
18861        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18862        ) {
18863            let mut rig = rig().await;
18864            rig.sync(vec![session(1)]).await;
18865            let mut route = rig
18866                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18867                .await;
18868            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18869            rig.sync(vec![session(1)]).await;
18870            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18871            assert!(rig.live(&route) && route.untouched());
18872
18873            // A call in flight on the route when another carrier is added. A
18874            // forwarded REQUEST holds one credit on the route's flow until the
18875            // module answers; the router takes it exactly like this.
18876            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18877                .forwarding
18878                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18879                .unwrap()
18880            else {
18881                panic!("the route is bound");
18882            };
18883            binding.flow.acquire_tagged(9, false).await.unwrap();
18884            let mut widened = session(1);
18885            widened.carriers.push(carrier(MAGIC, None));
18886            rig.sync(vec![widened]).await;
18887            assert!(rig.live(&route) && route.untouched());
18888            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18889            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18890            // The call's credit is still held on an open flow, so its answer
18891            // will be delivered: closing the route would have closed the flow.
18892            assert_eq!(binding.flow.in_flight(), 1);
18893            binding
18894                .flow
18895                .acquire_tagged(10, false)
18896                .await
18897                .expect("the flow is still open");
18898        }
18899
18900        /// A swap's superseded endpoint keeps its routes until drained; ending
18901        /// the scope closes them there too.
18902        #[tokio::test]
18903        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18904            let mut rig = rig().await;
18905            rig.sync(vec![session(1)]).await;
18906            let mut on_incumbent = rig
18907                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18908                .await;
18909
18910            // Swap plexus: register a candidate and cut over, leaving the
18911            // incumbent superseded with the route still on it.
18912            let (candidate, _candidate_rx) = wide_ctx(9);
18913            let registration = rig
18914                .handler
18915                .registry
18916                .register_candidate_with_control_ops(
18917                    manifest(PLEXUS, PROTOCOL_VERSION),
18918                    PROTOCOL_VERSION,
18919                    candidate.connection_id,
18920                    module_baseline_control_ops(),
18921                )
18922                .unwrap();
18923            rig.forwarding
18924                .register_candidate_module_connection(
18925                    candidate.connection_id,
18926                    PLEXUS.to_string(),
18927                    PROTOCOL_VERSION,
18928                    manifest_concurrency(&registration.manifest),
18929                    candidate.egress.clone(),
18930                )
18931                .unwrap();
18932            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18933            rig.handler
18934                .registry
18935                .promote_candidate(PLEXUS)
18936                .unwrap()
18937                .unwrap();
18938            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18939
18940            rig.sync(Vec::new()).await;
18941            assert!(!rig.live(&on_incumbent));
18942            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18943            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18944            let goodbye = incumbent_rx
18945                .try_recv()
18946                .expect("the superseded endpoint is told")
18947                .frame;
18948            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18949        }
18950    }
18951}
18952
18953#[cfg(test)]
18954mod concurrency_default_exposure_tests {
18955    use super::*;
18956
18957    fn hello_body(role_json: &str) -> Vec<u8> {
18958        format!(
18959            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18960        )
18961        .into_bytes()
18962    }
18963
18964    fn manifest_from(body: &[u8]) -> ModuleManifest {
18965        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18966        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18967            .expect("manifest parses")
18968    }
18969
18970    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18971
18972    #[test]
18973    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18974        let body = hello_body(&format!(
18975            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18976        ));
18977        let manifest = manifest_from(&body);
18978        // Precondition: serde really resolved it to the default, so the typed
18979        // manifest alone cannot answer the question this probe exists for.
18980        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18981        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18982    }
18983
18984    #[test]
18985    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18986        let body = hello_body(&format!(
18987            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18988        ));
18989        let manifest = manifest_from(&body);
18990        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18991        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18992    }
18993
18994    #[test]
18995    fn non_management_roles_are_never_reported() {
18996        let body = hello_body(
18997            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18998        );
18999        let manifest = manifest_from(&body);
19000        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19001    }
19002}