Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_confirm_operations,
25        validate_hello_event_declarations, validate_hello_flow_operations,
26        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29    },
30    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
31    scope::{
32        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
33        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
34        SCOPE_SYNC_OP,
35    },
36    session::{
37        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
38        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
39        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
40        ROLE_VERSIONS_FIELD,
41    },
42    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
43    Priority, RouteTarget, PROTOCOL_VERSION,
44};
45use tokio::time::{timeout_at, Instant};
46use tracing::{debug, info, instrument::WithSubscriber, warn};
47
48use crate::{
49    caller_identity::CallerIdentity,
50    capability_requirements::{
51        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
52        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
53        RequirementStatus, RuntimeModule,
54    },
55    child_roster::DaemonShutdownFlag,
56    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
57    forwarding::{
58        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
59        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
60        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
61    },
62    observability::{
63        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
64    },
65    privileged_prompt::{prompt_text, PrivilegedOperation},
66    provenance::{
67        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
68    },
69    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
70    router::{daemon_error_flags, RouteCtx, RouterError},
71    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
72    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
73    stderr_tail::{CaptureState, TailEntry},
74    supervise::{
75        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
76        SupervisorHandle, SwapHelloAdmission,
77    },
78    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
79};
80
81/// Lowest envelope version this subc build will negotiate.
82///
83/// Module HELLO negotiation is exact: peers must use the daemon's locked
84/// protocol version. Older and newer peers receive `version_unsupported` and
85/// are not registered.
86pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
87
88const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
89const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
90const CAP_PING_PONG: &str = "ping_pong_v1";
91const CAP_SESSION_ATTACH: &str = "session_attach_v1";
92const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
93
94const SUBC_CONTROL_OPS: &[&str] = &[
95    ops::DAEMON_SHUTDOWN_INTENT,
96    ops::ACTIVITY_RECORD,
97    ops::ACTIVITY_QUERY,
98    ops::ACTIVITY_SUBSCRIBE,
99    ops::SERVER_DESCRIBE,
100    ops::CATALOG_LIST,
101    ops::ROUTE_OPEN,
102    ops::ROUTE_POLL,
103    ops::ROUTE_CLOSING,
104    ops::ROUTE_CLOSED,
105    ops::SUPERVISOR_LIST,
106    ops::SUPERVISOR_RESTART,
107    ops::SUPERVISOR_SWAP,
108    ops::SUPERVISOR_RELOAD,
109    ops::SUPERVISOR_RESCAN,
110    ops::SUPERVISOR_RELEASE_RESERVED,
111    ops::SUPERVISOR_SET_ENABLED,
112    ops::SUPERVISOR_HEALTH_PROBE,
113    ops::SUPERVISOR_HEALTH,
114    ops::SUPERVISOR_STDERR_TAIL,
115    ops::SUPERVISOR_TERMINALS,
116    ops::SUPERVISOR_ROUTES,
117    ops::SUPERVISOR_PROVENANCE,
118    ops::SUPERVISOR_SPAWN_SNAPSHOT,
119    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
120];
121
122/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
123/// announced by a capability in `server.describe` instead, and clients check
124/// that capability before sending it. `ck daemon --json` prints `subc_ops`
125/// verbatim and scripts read that output, so a new name there would change it.
126/// A malformed body for one of these is still refused `invalid_control_body`
127/// rather than as an unknown op.
128const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
129
130mod rescan_checked;
131use rescan_checked::{RescanMode, RescanPlan};
132
133const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
134    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
135    "supervisor.live_roots",
136    SCOPE_SYNC_OP,
137    SCOPE_APPLY_OP,
138    SCOPE_DESCRIBE_OP,
139    "operator.confirm",
140];
141
142/// Module-originated ops the daemon answers but does not advertise in
143/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
144/// belongs to is incomplete, so no module is told it works before it does.
145const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
146
147const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
148
149/// How long subc waits for a module to ack a relayed route.bind before returning
150/// `module_timeout`. The ack waits on the module's own configure, which for AFT
151/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
152/// and DB-open work — on a cold page cache or a large repo that legitimately
153/// exceeds a couple of seconds. The default is generous because rejecting a VALID
154/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
155/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
156pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
157
158/// How many CONSECUTIVE full-budget relay timeouts against one target module
159/// open that module's bind-relay breaker.
160///
161/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
162/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
163/// so three consecutive full-budget timeouts take ~36s to observe: every client
164/// whose open contributed to opening the breaker had already given up on its
165/// own. That is what makes opening the breaker unable to turn a call that would
166/// have succeeded into a refusal — it can only make an already-failing module
167/// fail faster.
168///
169/// Two would be reachable inside one default deadline. One would convict a
170/// module on a single cold-cache bind, which is exactly the valid-but-slow case
171/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
172pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
173
174/// How long a module's bind-relay breaker stays open before exactly one
175/// `route.open` is let through as a probe.
176///
177/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
178/// re-pays a full-budget stall almost continuously, and the breaker stops being
179/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
180/// deadline: a client that starts retrying after the module recovers has to get
181/// a probe opportunity inside its own deadline, or the breaker converts a
182/// recovered module into a failed call — the failure it exists to prevent,
183/// pointed the other way.
184///
185/// 20s sits between those with room on both sides, and it caps what a wedged
186/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
187/// rather than one per `route.open` per connection. The stall that motivated
188/// this, with its measurements, is written up in
189/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
190/// waited the whole budget out.
191pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
192
193const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
194const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
195
196fn reload_verdict(
197    configured: &Path,
198    spawned_from: Option<&Path>,
199    image: subc_control::RunningImageAgreement,
200) -> PendingReloadVerdict {
201    let path = match spawned_from {
202        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
203            ReloadPathAgreement::Match
204        }
205        Some(spawned_from) => ReloadPathAgreement::Mismatch {
206            configured: configured.to_path_buf(),
207            spawned_from: spawned_from.to_path_buf(),
208        },
209        None => ReloadPathAgreement::Unavailable {
210            reason: if matches!(
211                image,
212                subc_control::RunningImageAgreement::Unavailable {
213                    reason: subc_control::RunningImageUnavailableReason::NotRunning
214                }
215            ) {
216                ReloadPathUnavailableReason::NotRunning
217            } else {
218                ReloadPathUnavailableReason::SpawnedPathUnavailable
219            },
220        },
221    };
222    PendingReloadVerdict {
223        path,
224        image,
225        spec: None,
226    }
227}
228
229fn launch_spec_agreement(
230    configured: &crate::supervise::ModuleSpec,
231    running: bool,
232    spawned: Option<&crate::supervise::LaunchSpecDigest>,
233) -> subc_control::LaunchSpecAgreement {
234    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
235    if !running {
236        return Agreement::Unavailable {
237            reason: Reason::NotRunning,
238        };
239    }
240    match spawned {
241        None => Agreement::Unavailable {
242            reason: Reason::SpawnedSpecUnavailable,
243        },
244        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
245            Agreement::Match
246        }
247        Some(_) => Agreement::Mismatch,
248    }
249}
250
251fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
252    if configured == spawned_from {
253        return true;
254    }
255    #[cfg(windows)]
256    {
257        // Windows spawn records the resolved verbatim path (\\?\...), while
258        // configuration keeps its original spelling. Resolve both with the OS
259        // instead of stripping prefixes or folding case, which could conflate
260        // distinct names in case-sensitive directories. Different resolved
261        // paths still differ even if their files have identical content.
262        match (
263            std::fs::canonicalize(configured),
264            std::fs::canonicalize(spawned_from),
265        ) {
266            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
267            _ => false,
268        }
269    }
270    #[cfg(not(windows))]
271    {
272        false
273    }
274}
275
276#[derive(Clone)]
277struct DaemonProvenanceFacts {
278    build: DaemonBuildProvenance,
279    pid: Option<u32>,
280    started_at_ms: Option<u64>,
281    start_clock: Option<crate::clock::StartClock>,
282    executable_path: Option<PathBuf>,
283    executable_identity: Option<SpawnedFileIdentity>,
284    process_start_time: Option<u64>,
285    probe: ExecutableIdentityProbe,
286}
287
288impl Default for DaemonProvenanceFacts {
289    fn default() -> Self {
290        Self {
291            build: DaemonBuildProvenance {
292                build_git_sha: None,
293                build_lock_digest: None,
294            },
295            pid: None,
296            started_at_ms: None,
297            start_clock: None,
298            executable_path: None,
299            executable_identity: None,
300            process_start_time: None,
301            probe: ExecutableIdentityProbe::default(),
302        }
303    }
304}
305
306#[derive(Debug, Clone)]
307struct SupervisorRescanContext {
308    supervisor: Supervisor,
309    config_path: PathBuf,
310    configured_port: Option<u16>,
311    storage_config: Option<crate::daemon_config::StorageConfig>,
312    admission_facts_carrier_module_id: Option<String>,
313    admission_facts_targets: Option<Vec<String>>,
314    scope_authority_owners: Vec<String>,
315    privileged_operator_prompt: PrivilegedOperatorPrompt,
316    /// Counts executed rescans that set out to change the supervised modules'
317    /// staged or applied configuration, counted before the first change so a
318    /// rescan that fails partway still counts. With the daemon incarnation it
319    /// names the state a `supervisor.rescan_checked` preview was computed
320    /// against.
321    generation: Arc<std::sync::atomic::AtomicU64>,
322}
323
324/// Refusal labels passed to `observe_route_open_refusal` that mean the target
325/// module is not serving right now, and so open or extend an outage in the
326/// route outage tracker. Every one of them is only reachable after the target
327/// was found in the registry, which is what keeps an arbitrary client-chosen
328/// id from ever creating tracker state.
329///
330/// Deliberately absent: `not_registered` and `removed` (the id may be
331/// anything a client sent, and a removed module is gone on purpose),
332/// `protocol_none` (such a module never serves routes, so nothing is out),
333/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
334/// capability and admission-facts refusals (they refuse the caller, not a
335/// module outage), and `relay_reservation_failed` (its code ranges over
336/// capacity limits as well as a vanished connection). Capacity, breaker,
337/// relay-timeout and module-rejection refusals do not pass through that
338/// function at all; the breaker logs its own transitions.
339///
340/// The two not-serving refusals that bypass that function record themselves
341/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
342/// `required_capability_unprovided` is not tracked: the module itself is up,
343/// and the outage belongs to the missing provider.
344const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
345    "reloading",
346    "supervisor_not_live",
347    "registration_not_active",
348    "no_forwarding_connection",
349    "relay_send_failed",
350];
351
352/// Real channel-0 control handler for subc itself.
353#[derive(Clone)]
354pub struct ControlHandler {
355    registry: Arc<Registry>,
356    activity: Option<crate::activity::ActivityLedger>,
357    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
358    forwarding: Arc<ForwardingTable>,
359    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
360    supervisor: SupervisorHandle,
361    subc_capabilities: Arc<[String]>,
362    /// Daemon-wide route.bind relay budget. Used as the fallback when the
363    /// target module has no per-module override in
364    /// `route_bind_relay_timeouts`.
365    route_bind_relay_timeout: Duration,
366    /// Per-module route.bind relay budget overrides, keyed by module id. When
367    /// `handle_route_open` resolves the deadline for a target module, a
368    /// per-module entry wins over the daemon-wide value above.
369    route_bind_relay_timeouts: BTreeMap<String, Duration>,
370    /// Per-target-module bind-relay breaker state. Shared with the forwarding
371    /// table, which is where a new module connection resets it.
372    route_bind_breakers: RouteBindBreakers,
373    /// Live relay admissions keyed by target module. Shared through the
374    /// forwarding table so cloned or separately built handlers enforce one cap.
375    route_bind_concurrency: RouteBindConcurrency,
376    /// Start and end of each module's not-serving period as seen by
377    /// `route.open`, so an outage gets one line at each edge instead of only
378    /// the per-refusal INFO lines. Taken from the forwarding table, so every
379    /// handler built over one table shares it.
380    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
381    /// Consecutive relay timeouts that open a module's breaker.
382    route_bind_breaker_threshold: u32,
383    /// How long a breaker stays open before one probe is admitted.
384    route_bind_breaker_cooldown: Duration,
385    health_probe_timeout: Duration,
386    /// Central storage policy. When set, each registering module receives its
387    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
388    storage_config: Option<crate::daemon_config::StorageConfig>,
389    /// The machine id established at boot, served on every HELLO_ACK and on
390    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
391    /// changes the file, never this value. `None` serves no id.
392    machine_id: Option<crate::machine_id::MachineId>,
393    admission_facts_carrier_module_id: Option<String>,
394    admission_facts_targets: Option<Vec<String>>,
395    /// Scope records with their sync authorities and tombstones; see
396    /// `crate::scopes`. Shared by clones of this handler, so every connection
397    /// reads and writes one table.
398    scopes: Arc<RwLock<ScopeTable>>,
399    /// Wakes the expiry loop when an accepted scope change may have added a
400    /// deadline, so the loop can wait without ticking while no live scope has
401    /// one. A permit is stored when nothing waits, so no wake is lost.
402    scope_deadline_added: Arc<tokio::sync::Notify>,
403    /// The configured `scope_authority_owners`, kept so a rescan can report a
404    /// changed value as needing a daemon restart; rescan never applies it.
405    scope_authority_owners: Vec<String>,
406    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
407    privileged_operator_prompt: PrivilegedOperatorPrompt,
408    /// The local and peer address of each accepted TCP client connection, read
409    /// from the daemon's own socket, so a privileged request can name its caller.
410    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
411    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
412    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
413    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
414    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
415    pub(crate) operator_apps:
416        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417    /// One launch entry per app, behind a leaf lock: no other lock is taken
418    /// while it is held. Entries are memory-only, so a daemon or
419    /// installed-service restart drops them and a relaunched daemon never
420    /// mistakes an older process for its own launch.
421    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
422    /// Serializes `app.open` launches: the liveness check in the decision and
423    /// the later publish must not interleave across concurrent calls, or two
424    /// calls could both launch and the second publish would orphan the first
425    /// process. Refusals and `already_running` answers share the same lock;
426    /// `app.open` is rare, so the cost is negligible.
427    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
428    // Portable wire samples exercise dispatch without starting an OS process.
429    #[cfg(test)]
430    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
431    #[cfg(test)]
432    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
433    /// Test-only Windows signature and flash hooks. Production builds carry
434    /// neither: with the publisher unset, the production table never reaches
435    /// verification, and the default attention hook only logs.
436    #[allow(dead_code)]
437    #[cfg(windows)]
438    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
439    #[allow(dead_code)]
440    #[cfg(windows)]
441    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
442    /// Test-only tracking fault hook: makes opening the tracking handle on
443    /// the launched pid fail, so a test can prove an untrackable launch is
444    /// stopped rather than left running.
445    #[allow(dead_code)]
446    #[cfg(windows)]
447    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
448    /// Test-only count of Windows spawn attempts, so a test can prove a
449    /// refused launch started no process. Unit tests only.
450    #[cfg(all(test, windows))]
451    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
452    #[cfg(test)]
453    caller_lookup: Option<TestCallerLookup>,
454    /// The launch nonce each module connection presented at HELLO, which is how
455    /// a `scope.sync` is matched to the owner's current launch.
456    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
457    rescan: Option<SupervisorRescanContext>,
458    connected_clients: ConnectedClients,
459    counters: DaemonCounters,
460    /// Why this daemon serves no local endpoint, when binding one failed. `None`
461    /// when the endpoint is served, and also on a handler that was never told.
462    local_endpoint_unavailable: Option<String>,
463    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
464    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
465    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
466    /// their usual severity (see [`log_requirement_events`]). A handler built
467    /// without one has a flag that is never set.
468    daemon_shutdown: DaemonShutdownFlag,
469    daemon_provenance: DaemonProvenanceFacts,
470    #[cfg(test)]
471    control_dispatch_delay: Option<Duration>,
472    #[cfg(test)]
473    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
474    /// Holds `route.open` after the route.bind is queued to the module and
475    /// before it is marked enqueued, so a test can make the module's reply (or
476    /// a client teardown) land exactly inside that window.
477    #[cfg(test)]
478    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
479    /// Holds a HELLO after its registry entry is reserved and before its
480    /// forwarding endpoint is installed, so a test can read the catalog and
481    /// open routes while the module is registered but not yet routable. A
482    /// blocking barrier because the HELLO handler is synchronous; the test
483    /// runs the HELLO on its own thread.
484    #[cfg(test)]
485    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
486}
487
488impl fmt::Debug for ControlHandler {
489    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
490        f.debug_struct("ControlHandler")
491            .field("registry", &self.registry)
492            .field("forwarding", &self.forwarding)
493            .field("process_liveness", &self.process_liveness.is_some())
494            .field("supervisor", &self.supervisor)
495            .field("subc_capabilities", &self.subc_capabilities)
496            .finish()
497    }
498}
499
500struct RouteOpenRequest {
501    target: RouteTarget,
502    identity: BindIdentity,
503    consumer_identity: Option<ConsumerIdentity>,
504    consumer_capabilities: Option<Vec<String>>,
505    role_versions: Option<BTreeMap<String, String>>,
506    busy_refusals: Option<bool>,
507    admission_facts: Option<serde_json::Value>,
508    scope: Option<ScopeSelector>,
509}
510
511#[derive(Clone, Copy)]
512enum RouteOpenPhase {
513    Admission,
514    ProjectRoot,
515    EgressReservation,
516    BindSend,
517    BindAckWait,
518}
519
520/// Store elapsed time for each route-open phase, including attempts that end
521/// in an early refusal or backend error. Admission also includes rechecks and
522/// commit bookkeeping performed between the other timed phases.
523struct RouteOpenTimings {
524    module_id: String,
525    principal: String,
526    admission: Duration,
527    project_root: Duration,
528    egress_reservation: Duration,
529    bind_send: Duration,
530    bind_ack_wait: Duration,
531    phase: Option<RouteOpenPhase>,
532    phase_started_at: Instant,
533}
534
535impl RouteOpenTimings {
536    fn new() -> Self {
537        Self {
538            module_id: String::new(),
539            // Refusals before attestation must not claim a verified principal.
540            principal: "unresolved".to_string(),
541            admission: Duration::ZERO,
542            project_root: Duration::ZERO,
543            egress_reservation: Duration::ZERO,
544            bind_send: Duration::ZERO,
545            bind_ack_wait: Duration::ZERO,
546            phase: Some(RouteOpenPhase::Admission),
547            phase_started_at: Instant::now(),
548        }
549    }
550
551    fn finish(&mut self) {
552        let elapsed = self.phase_started_at.elapsed();
553        match self.phase.take() {
554            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
555            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
556            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
557            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
558            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
559            None => {}
560        }
561    }
562
563    fn enter(&mut self, phase: RouteOpenPhase) {
564        self.finish();
565        self.phase = Some(phase);
566        self.phase_started_at = Instant::now();
567    }
568}
569
570struct RouteBindReservationGuard {
571    forwarding: Arc<ForwardingTable>,
572    endpoint: ModuleEndpointId,
573    relay_corr: u64,
574    armed: bool,
575}
576
577struct ModuleControlRpcGuard {
578    forwarding: Arc<ForwardingTable>,
579    endpoint: ModuleEndpointId,
580    corr: u64,
581    armed: bool,
582}
583
584impl ModuleControlRpcGuard {
585    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
586        Self {
587            forwarding,
588            endpoint,
589            corr,
590            armed: true,
591        }
592    }
593
594    fn disarm(&mut self) {
595        self.armed = false;
596    }
597}
598
599impl Drop for ModuleControlRpcGuard {
600    fn drop(&mut self) {
601        if self.armed {
602            let _ = self
603                .forwarding
604                .cancel_module_control_rpc(self.endpoint, self.corr);
605        }
606    }
607}
608
609impl RouteBindReservationGuard {
610    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
611        Self {
612            forwarding,
613            endpoint,
614            relay_corr,
615            armed: true,
616        }
617    }
618
619    fn release_and_disarm(&mut self) {
620        if !self.armed {
621            return;
622        }
623        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
624            self.endpoint,
625            self.relay_corr,
626            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
627        ) {
628            send_goodbye_target_best_effort(
629                &self.forwarding.counters(),
630                &target,
631                "canceled route.bind",
632            );
633        }
634        self.armed = false;
635    }
636
637    fn disarm(&mut self) {
638        self.armed = false;
639    }
640}
641
642impl Drop for RouteBindReservationGuard {
643    fn drop(&mut self) {
644        self.release_and_disarm();
645    }
646}
647
648/// Per-target-module circuit breaker around the `route.bind` relay.
649///
650/// The connection reader is serial per connection, so a module whose `on_bind`
651/// sits on the ack blocks every LATER frame on the connections that call it,
652/// including calls to unrelated modules. This does not make any module's bind
653/// fast; it stops the daemon paying the full budget again and again for a
654/// condition it has already observed.
655///
656/// State is keyed by TARGET MODULE and shared by every connection: a wedged
657/// module wedges everyone, so what one connection learned should protect the
658/// rest.
659///
660/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
661/// relay to that module has actually timed out, and is removed again when a
662/// relay is accepted or the module reconnects, so it cannot grow with traffic
663/// or with modules that behave.
664///
665/// # Why a `std` mutex here is not the head-of-line defect again
666///
667/// Acquisition never awaits. The critical section is a hash lookup plus a few
668/// integer updates, with no I/O and no `.await` inside it, so a reader task
669/// cannot be descheduled behind it the way it can behind
670/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
671/// primitive, held for the same kind of work, as the refusal counter this very
672/// path already increments.
673///
674/// It is also NOT on the data-plane splice path: only `route.open` and module
675/// registration touch it, so bound-route frames gain no state check and no
676/// contention.
677#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindBreakers {
679    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
680}
681
682#[derive(Debug, Clone, Default)]
683pub(crate) struct RouteBindConcurrency {
684    modules: Arc<Mutex<HashMap<String, usize>>>,
685}
686
687struct RouteBindConcurrencyGuard {
688    concurrency: RouteBindConcurrency,
689    module_id: String,
690}
691
692impl RouteBindConcurrency {
693    /// Admit without waiting. Waiting here would move the bind stall from the
694    /// module reply to a semaphore and restore reader head-of-line blocking.
695    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
696        let mut modules = self
697            .modules
698            .lock()
699            .expect("route.bind concurrency mutex poisoned");
700        let in_flight = modules.entry(module_id.to_string()).or_default();
701        if *in_flight >= limit {
702            return Err(*in_flight);
703        }
704        *in_flight += 1;
705        Ok(RouteBindConcurrencyGuard {
706            concurrency: self.clone(),
707            module_id: module_id.to_string(),
708        })
709    }
710}
711
712impl Drop for RouteBindConcurrencyGuard {
713    fn drop(&mut self) {
714        let mut modules = self
715            .concurrency
716            .modules
717            .lock()
718            .expect("route.bind concurrency mutex poisoned");
719        let remove = {
720            let in_flight = modules
721                .get_mut(&self.module_id)
722                .expect("admitted route.bind has a concurrency entry");
723            *in_flight -= 1;
724            *in_flight == 0
725        };
726        if remove {
727            modules.remove(&self.module_id);
728        }
729    }
730}
731
732#[derive(Debug, Default)]
733struct ModuleBreakerState {
734    /// Relay timeouts observed with no accepted relay in between.
735    consecutive_timeouts: u32,
736    /// `Some` while the breaker is open: the instant the cooldown expires and
737    /// the next arrival may probe. `None` means closed.
738    cooldown_until: Option<Instant>,
739    /// A half-open probe has been admitted and has not settled yet. This is
740    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
741    /// that read the cooldown, so concurrent opens arriving at the moment the
742    /// cooldown expires cannot all decide that they are the probe.
743    probe_in_flight: Option<Arc<()>>,
744}
745
746/// What the breaker decided for one `route.open`, before any relay work.
747enum RouteBindAdmission<'a> {
748    Admitted {
749        guard: RouteBindBreakerGuard<'a>,
750        /// This open is the single half-open probe, so the transition is worth
751        /// one log line.
752        probe: bool,
753    },
754    Refused {
755        consecutive_timeouts: u32,
756        /// What is left of the cooldown. Zero when the refusal is because the
757        /// one probe is already in flight rather than because the cooldown has
758        /// not elapsed.
759        retry_in: Duration,
760        probe_in_flight: bool,
761    },
762}
763
764/// An outstanding admission, which must be told how its relay settled.
765///
766/// `Drop` settles it as inconclusive, so an early return between admission and
767/// the relay -- or the whole handler being cancelled when the client
768/// disconnects -- releases a half-open probe slot instead of leaving the
769/// breaker wedged half-open with no further probes.
770struct RouteBindBreakerGuard<'a> {
771    breakers: RouteBindBreakers,
772    module_id: &'a str,
773    probe_token: Option<Arc<()>>,
774    settled: bool,
775}
776
777impl RouteBindBreakerGuard<'_> {
778    /// The module answered within the budget and took the bind. THE ONLY
779    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
780    /// breaker, which is a transition worth logging.
781    fn record_accepted(&mut self) -> bool {
782        self.settled = true;
783        self.breakers.record_accepted(self.module_id)
784    }
785
786    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
787    /// COUNTS TOWARD OPENING.
788    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
789        self.settled = true;
790        self.breakers.record_timeout(
791            self.module_id,
792            self.probe_token.as_ref(),
793            threshold,
794            cooldown,
795        )
796    }
797
798    /// Everything else: the module REJECTED the bind, its connection went away
799    /// mid-relay, or the waiter was cancelled.
800    ///
801    /// None of these is evidence that a module is slow, and each already has
802    /// its own refusal with its own code. A module that rejects a bind in
803    /// microseconds is healthy and must never be convicted for it; a module
804    /// that died has said nothing about the module that replaces it. So these
805    /// neither increment nor reset the count -- they only release a probe slot.
806    fn record_inconclusive(&mut self) {
807        self.settled = true;
808        self.breakers
809            .record_inconclusive(self.module_id, self.probe_token.as_ref());
810    }
811}
812
813impl Drop for RouteBindBreakerGuard<'_> {
814    fn drop(&mut self) {
815        if !self.settled {
816            self.breakers
817                .record_inconclusive(self.module_id, self.probe_token.as_ref());
818        }
819    }
820}
821
822/// The breaker moved to open, reported so the caller can log it outside the
823/// lock. Opening is rare and load-bearing; the refusals that follow are
824/// frequent and are counted rather than logged.
825struct BreakerOpened {
826    consecutive_timeouts: u32,
827    /// True when a failed probe re-opened an already-open breaker, which reads
828    /// very differently in a log from a first opening.
829    reopened_after_probe: bool,
830}
831
832impl RouteBindBreakers {
833    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
834        self.modules
835            .lock()
836            .expect("route.bind breaker mutex poisoned")
837    }
838
839    /// Decide whether this `route.open` may attempt its relay. Takes the map
840    /// lock and nothing else, and never awaits.
841    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
842        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
843            probe: probe_token.is_some(),
844            guard: RouteBindBreakerGuard {
845                breakers: self.clone(),
846                module_id,
847                probe_token,
848                settled: false,
849            },
850        };
851
852        let mut modules = self.lock();
853        let Some(state) = modules.get_mut(module_id) else {
854            return admitted(None);
855        };
856        let Some(cooldown_until) = state.cooldown_until else {
857            return admitted(None);
858        };
859        if state.probe_in_flight.is_some() {
860            return RouteBindAdmission::Refused {
861                consecutive_timeouts: state.consecutive_timeouts,
862                retry_in: Duration::ZERO,
863                probe_in_flight: true,
864            };
865        }
866        let now = Instant::now();
867        if now < cooldown_until {
868            return RouteBindAdmission::Refused {
869                consecutive_timeouts: state.consecutive_timeouts,
870                retry_in: cooldown_until - now,
871                probe_in_flight: false,
872            };
873        }
874        let token = Arc::new(());
875        state.probe_in_flight = Some(Arc::clone(&token));
876        admitted(Some(token))
877    }
878
879    fn record_accepted(&self, module_id: &str) -> bool {
880        self.lock()
881            .remove(module_id)
882            .is_some_and(|state| state.cooldown_until.is_some())
883    }
884
885    fn record_timeout(
886        &self,
887        module_id: &str,
888        probe_token: Option<&Arc<()>>,
889        threshold: u32,
890        cooldown: Duration,
891    ) -> Option<BreakerOpened> {
892        let mut modules = self.lock();
893        let state = modules.entry(module_id.to_string()).or_default();
894        let was_open = state.cooldown_until.is_some();
895        let was_probe = Self::owns_probe(state, probe_token);
896        if was_probe {
897            state.probe_in_flight = None;
898        }
899        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
900        if state.consecutive_timeouts < threshold {
901            return None;
902        }
903        state.cooldown_until = Some(Instant::now() + cooldown);
904        Some(BreakerOpened {
905            consecutive_timeouts: state.consecutive_timeouts,
906            reopened_after_probe: was_open && was_probe,
907        })
908    }
909
910    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
911        // A relay can finish after the breaker was reset or after it opened
912        // again and started a new probe. Only the guard whose token matches the
913        // active probe may release it, so a late relay never frees a newer probe.
914        state
915            .probe_in_flight
916            .as_ref()
917            .zip(token)
918            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
919    }
920
921    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
922        if let Some(state) = self.lock().get_mut(module_id) {
923            if Self::owns_probe(state, probe_token) {
924                state.probe_in_flight = None;
925            }
926        }
927    }
928
929    /// Discard what was learned about a module, because the process it was
930    /// learned about is gone. Returns the discarded count when it was non-zero.
931    ///
932    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
933    /// `module_id` is a configuration identity that outlives any particular
934    /// child; what the breaker observed was the process behind the module
935    /// connection of the moment. When a new connection registers under that id
936    /// the verdict's subject no longer exists, so the verdict is stale by
937    /// construction rather than merely likely to be wrong. Keeping it would
938    /// apply a dead process's record to a live one, which is the same defect
939    /// class this breaker exists to stop the daemon committing.
940    ///
941    /// A half-open probe in flight is discarded with the rest: it was a
942    /// question about the old process.
943    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
944        self.lock()
945            .remove(module_id)
946            .map(|state| state.consecutive_timeouts)
947            .filter(|discarded| *discarded > 0)
948    }
949
950    /// Open breakers, for the `server.describe` counters object. `None` when
951    /// none is open, so the key stays absent rather than present-and-empty.
952    ///
953    /// This is the operator's answer to "is this module refusing instantly or
954    /// is it fine?", which look identical from a client that retries and then
955    /// succeeds.
956    fn open_snapshot(&self) -> Option<serde_json::Value> {
957        let now = Instant::now();
958        let modules = self.lock();
959        let open = modules
960            .iter()
961            .filter_map(|(module_id, state)| {
962                let cooldown_until = state.cooldown_until?;
963                Some((
964                    module_id.clone(),
965                    serde_json::json!({
966                        "consecutive_timeouts": state.consecutive_timeouts,
967                        "cooldown_remaining_ms":
968                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
969                        "probe_in_flight": state.probe_in_flight.is_some(),
970                    }),
971                ))
972            })
973            .collect::<serde_json::Map<String, serde_json::Value>>();
974        (!open.is_empty()).then_some(serde_json::Value::Object(open))
975    }
976}
977
978impl ControlHandler {
979    pub fn new(registry: Arc<Registry>) -> Self {
980        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
981    }
982
983    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
984        let counters = forwarding.counters();
985        // Taken from the forwarding table rather than created here, so that the
986        // breaker a `route.open` consults is the same one a module's
987        // registration resets, however many handlers are built over one table.
988        let route_bind_breakers = forwarding.route_bind_breakers();
989        let route_bind_concurrency = forwarding.route_bind_concurrency();
990        let route_outages = forwarding.route_outages();
991        Self {
992            registry,
993            activity: None,
994            activity_streams: Arc::new(Mutex::new(HashMap::new())),
995            forwarding,
996            process_liveness: None,
997            supervisor: SupervisorHandle::new(),
998            subc_capabilities: Arc::from([
999                CAP_MANIFEST_REGISTRATION.to_string(),
1000                CAP_CHANNEL_LIFECYCLE.to_string(),
1001                CAP_PING_PONG.to_string(),
1002                CAP_SESSION_ATTACH.to_string(),
1003                CAP_ADMISSION_FACTS_RELAY.to_string(),
1004                CAP_SCOPES_V1.to_string(),
1005                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1006                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1007                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1008                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1009                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1010                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1011                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1012                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1013                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1014                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1015                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1016            ]),
1017            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1018            route_bind_relay_timeouts: BTreeMap::new(),
1019            route_bind_breakers,
1020            route_bind_concurrency,
1021            route_outages,
1022            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1023            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1024            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1025            storage_config: None,
1026            machine_id: None,
1027            admission_facts_carrier_module_id: None,
1028            admission_facts_targets: None,
1029            scopes: Arc::new(RwLock::new(ScopeTable::new(
1030                crate::daemon_config::default_scope_authority_owners(),
1031            ))),
1032            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1033            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1034            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1035            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1036            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1037            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1038            local_connections: Arc::new(Mutex::new(HashSet::new())),
1039            operator_app_policy: Default::default(),
1040            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1041            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1042            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1043            #[cfg(test)]
1044            app_open_test_platform: None,
1045            #[cfg(test)]
1046            app_open_test_launch: None,
1047            #[cfg(windows)]
1048            windows_verify: None,
1049            #[cfg(windows)]
1050            windows_flash: None,
1051            #[cfg(windows)]
1052            windows_track: None,
1053            #[cfg(all(test, windows))]
1054            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1055            #[cfg(test)]
1056            caller_lookup: None,
1057            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1058            rescan: None,
1059            connected_clients: ConnectedClients::new(),
1060            local_endpoint_unavailable: None,
1061            counters,
1062            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1063            daemon_shutdown: DaemonShutdownFlag::default(),
1064            daemon_provenance: DaemonProvenanceFacts::default(),
1065            #[cfg(test)]
1066            control_dispatch_delay: None,
1067            #[cfg(test)]
1068            provenance_probe_override: None,
1069            #[cfg(test)]
1070            route_bind_relay_sent_hold: None,
1071            #[cfg(test)]
1072            hello_registry_to_forwarding_hold: None,
1073        }
1074    }
1075
1076    /// Set the central storage policy: registering modules then receive their
1077    /// resolved storage descriptor in HELLO_ACK.
1078    pub fn with_storage_config(
1079        mut self,
1080        storage_config: Option<crate::daemon_config::StorageConfig>,
1081    ) -> Self {
1082        self.storage_config = storage_config;
1083        self
1084    }
1085
1086    /// Open the ledger on its own writer thread; an unavailable sink does not
1087    /// prevent the daemon from serving routing or supervisor requests.
1088    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1089        self.activity = Some(crate::activity::ActivityLedger::open(
1090            run_dir.join("activity.jsonl"),
1091            incarnation,
1092        ));
1093        self.supervisor.set_activity(self.activity.clone());
1094        self
1095    }
1096
1097    /// Set the machine id served to every registering module (HELLO_ACK) and on
1098    /// `server.describe`.
1099    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1100        self.machine_id = machine_id;
1101        self
1102    }
1103
1104    /// Configure the exact reserved module and target ids permitted to relay
1105    /// opaque admission facts. Config-file loading validates this authority;
1106    /// this builder keeps the same policy available to embedded test daemons.
1107    pub fn with_admission_facts_config(
1108        mut self,
1109        carrier_module_id: Option<String>,
1110        targets: Option<Vec<String>>,
1111    ) -> Self {
1112        self.admission_facts_carrier_module_id = carrier_module_id;
1113        self.admission_facts_targets = targets;
1114        self
1115    }
1116
1117    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1118    /// Replaces the scope table with an empty one under the new list, so call it
1119    /// while building the handler, before any module can sync.
1120    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1121        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1122        self.scope_authority_owners = owners;
1123        self
1124    }
1125
1126    /// Set the startup policy before installing the rescan context or serving requests.
1127    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1128        self.privileged_operator_prompt = mode;
1129        self
1130    }
1131
1132    /// Override the route.bind relay timeout. Used by tests that assert the
1133    /// timeout path so they don't block on the production-safe default.
1134    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1135        self.route_bind_relay_timeout = timeout;
1136        self
1137    }
1138
1139    /// Install per-module route.bind relay budget overrides. A module id
1140    /// listed here wins over the daemon-wide default set via
1141    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1142    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1143    /// the same `Duration` the bind path will use.
1144    pub fn with_route_bind_relay_timeouts(
1145        mut self,
1146        timeouts: impl IntoIterator<Item = (String, Duration)>,
1147    ) -> Self {
1148        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1149        self
1150    }
1151
1152    /// Resolve the route.bind relay budget for a specific target module id.
1153    /// Per-module overrides win; the daemon-wide value (set via
1154    /// `with_route_bind_relay_timeout` or the built-in default) is the
1155    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1156    /// the same resolution `handle_route_open` will use.
1157    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1158        self.route_bind_relay_timeouts
1159            .get(module_id)
1160            .copied()
1161            .unwrap_or(self.route_bind_relay_timeout)
1162    }
1163
1164    /// Override the per-module bind-relay breaker policy.
1165    ///
1166    /// Used by tests, which cannot spend three production budgets opening a
1167    /// breaker or twenty seconds waiting for its cooldown. The production
1168    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1169    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1170    /// reasoning for the numbers.
1171    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1172        self.route_bind_breaker_threshold = threshold.max(1);
1173        self.route_bind_breaker_cooldown = cooldown;
1174        self
1175    }
1176
1177    #[cfg(test)]
1178    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1179        self.health_probe_timeout = timeout;
1180        self
1181    }
1182
1183    #[cfg(test)]
1184    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1185        self.control_dispatch_delay = Some(delay);
1186        self
1187    }
1188
1189    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1190    /// announce it is parked, once to be released. Build the barrier for two
1191    /// parties, the test being the other one.
1192    #[cfg(test)]
1193    pub(crate) fn with_route_bind_relay_sent_hold(
1194        mut self,
1195        hold: Arc<tokio::sync::Barrier>,
1196    ) -> Self {
1197        self.route_bind_relay_sent_hold = Some(hold);
1198        self
1199    }
1200
1201    /// A HELLO waits on `hold` twice between reserving its registry entry and
1202    /// installing its forwarding endpoint: once to announce it is parked, once
1203    /// to be released. Build the barrier for two parties, the test being the
1204    /// other one, and run the HELLO on a thread of its own.
1205    #[cfg(test)]
1206    pub(crate) fn with_hello_registry_to_forwarding_hold(
1207        mut self,
1208        hold: Arc<std::sync::Barrier>,
1209    ) -> Self {
1210        self.hello_registry_to_forwarding_hold = Some(hold);
1211        self
1212    }
1213
1214    #[cfg(test)]
1215    fn hold_between_registry_and_forwarding(&self) {
1216        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1217            hold.wait();
1218            hold.wait();
1219        }
1220    }
1221
1222    pub fn with_process_liveness(
1223        mut self,
1224        process_liveness: Arc<dyn ModuleProcessLiveness>,
1225    ) -> Self {
1226        self.process_liveness = Some(process_liveness);
1227        self
1228    }
1229
1230    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1231        self.supervisor = supervisor;
1232        self.supervisor.set_activity(self.activity.clone());
1233        self
1234    }
1235
1236    pub fn with_daemon_provenance(
1237        mut self,
1238        pid: u32,
1239        started_at_ms: u64,
1240        executable_path: Option<PathBuf>,
1241        build_git_sha: Option<String>,
1242        build_lock_digest: Option<String>,
1243    ) -> Self {
1244        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1245        let process_start_time = process_start_time(pid);
1246        self.daemon_provenance = DaemonProvenanceFacts {
1247            build: DaemonBuildProvenance {
1248                build_git_sha,
1249                build_lock_digest,
1250            },
1251            pid: Some(pid),
1252            started_at_ms: Some(started_at_ms),
1253            start_clock: None,
1254            executable_path,
1255            executable_identity,
1256            process_start_time,
1257            probe: ExecutableIdentityProbe::default(),
1258        };
1259        self
1260    }
1261
1262    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1263        self.daemon_provenance.start_clock = Some(clock);
1264        self
1265    }
1266
1267    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1268    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1269        self.scopes
1270            .write()
1271            .unwrap_or_else(|poisoned| poisoned.into_inner())
1272            .set_wall_clock(Arc::new(clock));
1273        self
1274    }
1275
1276    #[cfg(test)]
1277    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1278        self.provenance_probe_override = Some(result);
1279        self
1280    }
1281
1282    /// Install the configured module set and its reserved capability bindings.
1283    /// Bindings are configuration-scoped and may point at a provider that has not
1284    /// been installed yet, so this does not require the bound module to exist.
1285    pub fn with_capability_config(
1286        self,
1287        modules: impl IntoIterator<Item = (String, bool)>,
1288        reserved_capabilities: BTreeMap<String, String>,
1289    ) -> Self {
1290        self.capability_evaluator
1291            .configure(modules, reserved_capabilities);
1292        self
1293    }
1294
1295    /// Share the supervisor's daemon-shutdown flag with this handler, so
1296    /// requirement changes after the daemon has begun its announced shutdown
1297    /// are not reported as faults.
1298    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1299        self.daemon_shutdown = flag;
1300        self
1301    }
1302
1303    pub fn with_supervisor_rescan(
1304        mut self,
1305        supervisor: Supervisor,
1306        config_path: impl Into<PathBuf>,
1307        configured_port: Option<u16>,
1308    ) -> Self {
1309        self.rescan = Some(SupervisorRescanContext {
1310            supervisor,
1311            config_path: config_path.into(),
1312            configured_port,
1313            storage_config: self.storage_config.clone(),
1314            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1315            admission_facts_targets: self.admission_facts_targets.clone(),
1316            scope_authority_owners: self.scope_authority_owners.clone(),
1317            privileged_operator_prompt: self.privileged_operator_prompt,
1318            generation: Arc::default(),
1319        });
1320        self
1321    }
1322
1323    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1324        self.connected_clients = connected_clients;
1325        self
1326    }
1327
1328    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1329    /// an operator can see a bind failure without reading the daemon log.
1330    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1331        self.local_endpoint_unavailable = reason;
1332        self
1333    }
1334
1335    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1336        Arc::clone(&self.forwarding)
1337    }
1338
1339    pub(crate) fn counters(&self) -> DaemonCounters {
1340        self.counters.clone()
1341    }
1342
1343    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1344    /// requirement event without depending on an operator polling a status command.
1345    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1346        tokio::spawn(async move {
1347            loop {
1348                self.capability_evaluator
1349                    .wait_for_change_or_deadline()
1350                    .await;
1351                self.refresh_capability_requirements();
1352            }
1353        });
1354    }
1355
1356    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1357    /// deadline passed during sleep is enforced on the first tick after wake.
1358    /// While no live scope carries a deadline the loop waits for an accepted
1359    /// scope change instead of ticking, so a daemon without deadlines never
1360    /// wakes for expiry.
1361    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1362        tokio::spawn(async move {
1363            loop {
1364                let has_deadlines = self
1365                    .scopes
1366                    .read()
1367                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1368                    .has_deadlines();
1369                if !has_deadlines {
1370                    self.scope_deadline_added.notified().await;
1371                    continue;
1372                }
1373                tokio::time::sleep(Duration::from_secs(1)).await;
1374                if let Err(error) = self.sweep_expired_scopes() {
1375                    warn!(%error, "scope expiry sweep failed");
1376                }
1377            }
1378        });
1379    }
1380
1381    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1382        let mut table = self
1383            .scopes
1384            .write()
1385            .unwrap_or_else(|poisoned| poisoned.into_inner());
1386        let swept = table.sweep_expired();
1387        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1388        drop(table);
1389        self.close_scope_drained_routes(drained);
1390        Ok(())
1391    }
1392
1393    fn publish_scope_changes(
1394        &self,
1395        changes: &[crate::scopes::ScopeTagChange],
1396        expired: &[crate::scopes::ScopeExpired],
1397    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1398        if changes.is_empty() {
1399            return Ok(Vec::new());
1400        }
1401        let drained = self
1402            .forwarding
1403            .publish_scope_changes(changes)
1404            .map_err(RouterError::Forwarding)?;
1405        let mut counts = HashMap::new();
1406        for route in &drained {
1407            *counts
1408                .entry((
1409                    route.scope.owner.as_str(),
1410                    route.scope.scope_ref.as_str(),
1411                    route.scope.tag.scope_epoch,
1412                ))
1413                .or_insert(0usize) += 1;
1414        }
1415        for scope in expired {
1416            let routes_closed = counts
1417                .get(&(
1418                    scope.owner.as_str(),
1419                    scope.scope_ref.as_str(),
1420                    scope.scope_epoch,
1421                ))
1422                .copied()
1423                .unwrap_or(0);
1424            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1425                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1426                routes_closed, "scope expired");
1427        }
1428        Ok(drained)
1429    }
1430
1431    fn runtime_capability_snapshot(
1432        &self,
1433    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1434        let runtime = self
1435            .supervisor
1436            .list()
1437            .into_iter()
1438            .map(|module| {
1439                let status = module.status().map_err(|err| {
1440                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1441                })?;
1442                Ok(RuntimeModule {
1443                    module_id: status.module_id,
1444                    state: status.state,
1445                    enabled: status.enabled,
1446                })
1447            })
1448            .collect::<Result<Vec<_>, RouterError>>()?;
1449        // Include every registration, published or not. The capability
1450        // evaluator counts a module as providing its capabilities as soon as it
1451        // has registered, even before `catalog.list` shows it, so a HELLO in
1452        // progress does not change other modules' required-capability
1453        // verdicts. See `Registry::list_registered_modules`.
1454        let registrations = self.registry.list_registered_modules().map_err(|err| {
1455            RouterError::backend(
1456                0,
1457                0,
1458                format!("failed to list capability registrations: {err}"),
1459            )
1460        })?;
1461        let registrations = registrations
1462            .into_iter()
1463            .map(|registration| RegisteredModule {
1464                module_id: registration.manifest.module_id,
1465                module_version: registration.manifest.module_version,
1466                capabilities: registration.manifest.capabilities,
1467            })
1468            .collect();
1469        Ok((runtime, registrations))
1470    }
1471
1472    /// The capability side effects of a module becoming the active registration
1473    /// for its id: cache its manifest (logging how it differs from the cached
1474    /// one), run the deny census when its declarations call for one, and recompute the
1475    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1476    /// candidate's does not, and the supervisor does it at promotion instead,
1477    /// through [`crate::supervise::SwapPromotionObserver`].
1478    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1479        let cached_registration = RegisteredModule {
1480            module_id: registration.manifest.module_id.clone(),
1481            module_version: registration.manifest.module_version.clone(),
1482            capabilities: registration.manifest.capabilities.clone(),
1483        };
1484        let change = self.capability_evaluator.record_hello(&cached_registration);
1485        log_manifest_change(&cached_registration.module_id, &change);
1486        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1487            self.enforce_capability_denies();
1488        }
1489        self.refresh_capability_requirements();
1490    }
1491
1492    /// Point the shared supervisor handle at this handler for swap promotions.
1493    /// Called wherever a handler is put behind the `Arc` the router serves, so
1494    /// it can be held weakly.
1495    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1496        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1497            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1498        self.supervisor.set_swap_promotion_observer(observer);
1499    }
1500
1501    pub fn refresh_capability_requirements(&self) {
1502        match self.runtime_capability_snapshot() {
1503            Ok((runtime, registrations)) => {
1504                log_requirement_events(
1505                    self.capability_evaluator
1506                        .evaluate_now(&runtime, &registrations),
1507                    &self.daemon_shutdown,
1508                );
1509            }
1510            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1511        }
1512    }
1513
1514    /// Reconcile only live, attested route bindings after a capability deny edge
1515    /// or target claim was added. This is deliberately a control-plane census:
1516    /// the opaque forwarding hot path must not grow a per-frame capability check.
1517    fn enforce_capability_denies(&self) {
1518        let registrations = match self.registry.list_registered_modules() {
1519            Ok(snapshot) => snapshot,
1520            Err(err) => {
1521                warn!(error = %err, "failed to read registrations for capability deny census");
1522                return;
1523            }
1524        };
1525        let manifests = registrations
1526            .into_iter()
1527            .map(|registration| {
1528                (
1529                    registration.manifest.module_id.clone(),
1530                    registration.manifest,
1531                )
1532            })
1533            .collect::<BTreeMap<_, _>>();
1534        let census = match self.forwarding.route_census(None) {
1535            Ok(census) => census,
1536            Err(err) => {
1537                warn!(error = %err, "failed to read route census for capability deny enforcement");
1538                return;
1539            }
1540        };
1541
1542        for (target_module_id, routes) in census {
1543            let Some(target_manifest) = manifests.get(&target_module_id) else {
1544                continue;
1545            };
1546            let mut closed_routes = Vec::new();
1547            let mut goodbyes = Vec::new();
1548            for route in routes {
1549                let Principal::Reserved {
1550                    module_id: opening_module_id,
1551                } = &route.principal
1552                else {
1553                    continue;
1554                };
1555                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1556                    continue;
1557                };
1558                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1559                    continue;
1560                };
1561
1562                match self.forwarding.release_client_route(
1563                    route.goodbye_target.connection_id,
1564                    route.goodbye_target.channel,
1565                    route.goodbye_target.epoch,
1566                ) {
1567                    Ok(RouteRelease::Removed(module_goodbye)) => {
1568                        warn!(
1569                            opening_module_id,
1570                            target_module_id,
1571                            capability,
1572                            "force-closing route because an attested capability deny edge now matches"
1573                        );
1574                        goodbyes.push(module_goodbye);
1575                        goodbyes.push(route.goodbye_target.clone());
1576                        closed_routes.push(route);
1577                    }
1578                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1579                    Err(err) => warn!(
1580                        opening_module_id,
1581                        target_module_id,
1582                        capability,
1583                        error = %err,
1584                        "failed to force-close capability-denied route"
1585                    ),
1586                }
1587            }
1588
1589            if closed_routes.is_empty() {
1590                continue;
1591            }
1592            send_route_control_pushes(
1593                &self.forwarding,
1594                closed_routes,
1595                ClientControlPush::RouteClosed {
1596                    module_id: target_module_id,
1597                    channels: Vec::new(),
1598                    reason: RouteCloseReason::CapabilityDenied,
1599                    drained: false,
1600                    abandoned: 0,
1601                    excluded_subscriptions: 0,
1602                    terminal: Some(false),
1603                },
1604            );
1605            // route.closed records the reason; the route GOODBYE ends the
1606            // client's handle and settles any calls still pending on it.
1607            self.emit_route_goodbyes(goodbyes);
1608        }
1609    }
1610
1611    /// Why a registered module is not accepting new route binds, or `None` when
1612    /// it is. This is the module's effective readiness: its declared readiness
1613    /// first, then every `need: required` capability it declares evaluating to
1614    /// `provided`. `route.open` and `catalog.list` both read it here so the
1615    /// catalog never reports a module routable that `route.open` would refuse.
1616    fn not_ready_reason(
1617        &self,
1618        registration: &crate::registry::ModuleRegistration,
1619    ) -> Option<NotReadyReason> {
1620        if !registration.ready {
1621            return Some(NotReadyReason {
1622                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1623                capability: None,
1624            });
1625        }
1626        self.first_unprovided_required_capability(registration)
1627            .map(|capability| NotReadyReason {
1628                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1629                capability: Some(capability),
1630            })
1631    }
1632
1633    /// The lexicographically first capability this registration declares
1634    /// `need: required` whose evaluator verdict is not `provided`.
1635    ///
1636    /// The verdicts are the capability evaluator's own; nothing here decides
1637    /// what "provided" means. The evaluator counts a capability provided as
1638    /// soon as a module claiming it has REGISTERED, not once that module is
1639    /// ready. That distinction is what keeps two modules that require each
1640    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1641    /// is ready", each would wait for the other to become ready first and
1642    /// neither ever would. Do not tighten it to readiness.
1643    ///
1644    /// A required capability with no verdict at all means this registration's
1645    /// HELLO or catalog.update landed after the last recompute; recompute once
1646    /// rather than let a missing verdict read as either answer. If it is still
1647    /// missing (the recompute itself failed) the capability counts as
1648    /// unprovided: the refusal is retryable, and routing a module whose
1649    /// required provider is unknown is the outcome this check exists to stop.
1650    fn first_unprovided_required_capability(
1651        &self,
1652        registration: &crate::registry::ModuleRegistration,
1653    ) -> Option<String> {
1654        let required = registration
1655            .manifest
1656            .capabilities
1657            .iter()
1658            .flat_map(|declarations| declarations.requires.iter())
1659            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1660            .map(|requirement| requirement.capability.as_str())
1661            .collect::<BTreeSet<_>>();
1662        if required.is_empty() {
1663            return None;
1664        }
1665        let module_id = registration.manifest.module_id.as_str();
1666        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1667        if required
1668            .iter()
1669            .any(|capability| verdict(capability).is_none())
1670        {
1671            self.refresh_capability_requirements();
1672        }
1673        required
1674            .into_iter()
1675            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1676            .map(str::to_string)
1677    }
1678
1679    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1680        self.capability_evaluator
1681            .statuses()
1682            .into_iter()
1683            .map(capability_requirement_status)
1684            .collect()
1685    }
1686
1687    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1688    /// registration-release watch. The signal is what the supervisor waits on
1689    /// before spawning a replacement, so it must only fire once forwarding
1690    /// teardown is also done (see [`Self::cleanup_connection`] /
1691    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1692    /// admitted but its forwarding endpoint could not be installed.
1693    fn deregister_connection(
1694        &self,
1695        connection_id: ConnectionId,
1696        reason: RegistrationEndReason,
1697    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1698        self.registry
1699            .deregister_connection_with_reason(connection_id, reason)
1700    }
1701
1702    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1703        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1704            return None;
1705        }
1706        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1707            parse_client_control_request(&frame.body)
1708        else {
1709            return None;
1710        };
1711        Some(target_module_id(&target).to_string())
1712    }
1713
1714    pub(crate) fn route_open_capacity_refusal(
1715        &self,
1716        ctx: &RouteCtx,
1717        frame: &Frame,
1718        target_module_id: &str,
1719        in_flight: usize,
1720        limit: usize,
1721    ) -> Result<Frame, RouterError> {
1722        // Sent before the request reaches `handle_route_open`, so the caller
1723        // fields come from the raw frame here.
1724        let caller = self.route_open_caller_from_frame(frame);
1725        self.route_open_admission_refusal_frame(
1726            ctx,
1727            &caller,
1728            frame,
1729            target_module_id,
1730            "open_admission_full",
1731            (in_flight, limit),
1732            format!(
1733                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1734            ),
1735        )
1736    }
1737
1738    fn route_open_target_capacity_refusal(
1739        &self,
1740        ctx: &RouteCtx,
1741        caller: &RouteOpenCaller,
1742        frame: &Frame,
1743        target_module_id: &str,
1744        in_flight: usize,
1745    ) -> Result<Frame, RouterError> {
1746        self.route_open_admission_refusal_frame(
1747            ctx,
1748            caller,
1749            frame,
1750            target_module_id,
1751            "target_binds_full",
1752            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1753            format!(
1754                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1755            ),
1756        )
1757    }
1758
1759    /// Admission pressure clears as existing binds settle, so its refusal must
1760    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1761    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1762    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1763    /// cannot currently reach its target; `module_timeout` would falsely claim
1764    /// that a wait expired. A new, cleaner code would be terminal to deployed
1765    /// clients, so it requires a client-tolerance rollout before daemon emission.
1766    #[allow(clippy::too_many_arguments)]
1767    fn route_open_admission_refusal_frame(
1768        &self,
1769        ctx: &RouteCtx,
1770        caller: &RouteOpenCaller,
1771        frame: &Frame,
1772        target_module_id: &str,
1773        reason: &'static str,
1774        (in_flight, limit): (usize, usize),
1775        message: impl Into<String>,
1776    ) -> Result<Frame, RouterError> {
1777        let code = error_codes::TARGET_UNAVAILABLE;
1778        self.counters.increment_route_open_refused(code);
1779        info!(
1780            target: "control",
1781            code,
1782            reason,
1783            module_id = ?target_module_id,
1784            connection_id = ctx.connection_id.get(),
1785            principal = %caller.principal,
1786            scope = %caller.scope,
1787            harness = %caller.harness,
1788            in_flight,
1789            limit,
1790            "route.open refused"
1791        );
1792        control_error_frame(frame, code, message.into())
1793    }
1794
1795    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1796    ///
1797    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1798    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1799    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1800    #[cfg(test)]
1801    pub fn handle_control(
1802        &self,
1803        connection_id: ConnectionId,
1804        frame: Frame,
1805    ) -> Result<Vec<Frame>, RouterError> {
1806        match frame.header.ty {
1807            FrameType::Ping => Ok(vec![pong(&frame)?]),
1808            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1809            FrameType::Goodbye => self.handle_goodbye(connection_id),
1810            ty => Ok(vec![control_error_frame(
1811                &frame,
1812                "unsupported_control_frame",
1813                format!("unsupported channel-0 frame {ty:?}"),
1814            )?]),
1815        }
1816    }
1817
1818    pub async fn handle_control_frame(
1819        &self,
1820        ctx: &RouteCtx,
1821        frame: Frame,
1822    ) -> Result<Vec<Frame>, RouterError> {
1823        self.handle_control_frame_timed(ctx, frame, None).await
1824    }
1825
1826    #[cfg(any(test, feature = "test-support"))]
1827    pub fn with_test_operator_app_policy(
1828        mut self,
1829        policy: crate::operator_apps::TestOperatorAppPolicy,
1830    ) -> Self {
1831        self.operator_app_policy = policy.into();
1832        self
1833    }
1834
1835    /// Inject the Windows signature check for tests. The injected check
1836    /// answers the verified version string or the rejection reason; the
1837    /// daemon enforces the floor itself on the returned version.
1838    #[cfg(all(windows, any(test, feature = "test-support")))]
1839    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1840        self.windows_verify = Some(verify);
1841        self
1842    }
1843
1844    /// Record which pids the daemon would flash for `already_running`.
1845    #[cfg(all(windows, any(test, feature = "test-support")))]
1846    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1847        self.windows_flash = Some(flash);
1848        self
1849    }
1850
1851    /// Inject a tracking failure for tests: makes `track_launched_process`
1852    /// fail so a test can prove a launch that cannot be tracked is stopped,
1853    /// never left running.
1854    #[cfg(all(windows, any(test, feature = "test-support")))]
1855    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1856        self.windows_track = Some(track);
1857        self
1858    }
1859
1860    pub(crate) fn record_captured_peer(
1861        &self,
1862        connection: ConnectionId,
1863        peer: subc_os::local_peer::CapturedPeer,
1864    ) {
1865        self.record_observed_peer(connection, peer.observation().clone());
1866        #[cfg(windows)]
1867        if let Some(stamp) = self.stamp_windows_launch(&peer) {
1868            self.operator_apps
1869                .lock()
1870                .unwrap_or_else(|p| p.into_inner())
1871                .insert(connection, stamp);
1872        }
1873        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1874            self.operator_apps
1875                .lock()
1876                .unwrap_or_else(|p| p.into_inner())
1877                .insert(connection, stamp);
1878        }
1879    }
1880
1881    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1882        self.local_connections
1883            .lock()
1884            .unwrap_or_else(|p| p.into_inner())
1885            .insert(connection);
1886    }
1887
1888    // Keep the connection-level delivery decision shared with route.bind. The
1889    // target's capability opt-in is separate from recognition of the caller.
1890    fn connection_operator_app(
1891        &self,
1892        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1893    ) -> Option<subc_protocol::peer::OperatorApp> {
1894        stamp
1895            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1896            .map(|stamp| stamp.app.clone())
1897    }
1898
1899    fn handle_connection_describe(
1900        &self,
1901        ctx: &RouteCtx,
1902        frame: Frame,
1903    ) -> Result<Vec<Frame>, RouterError> {
1904        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1905            return Ok(vec![control_error_frame(
1906                &frame,
1907                "invalid_control_body",
1908                "malformed connection.describe body",
1909            )?]);
1910        }
1911        let local = self
1912            .local_connections
1913            .lock()
1914            .unwrap_or_else(|p| p.into_inner())
1915            .contains(&ctx.connection_id);
1916        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1917        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1918        Ok(vec![control_response_body_frame(
1919            &frame,
1920            &subc_control::ConnectionDescribeReply::new(local, operator_app),
1921            ops::CONNECTION_DESCRIBE,
1922        )?])
1923    }
1924
1925    pub(crate) fn record_observed_peer(
1926        &self,
1927        connection: ConnectionId,
1928        peer: subc_os::local_peer::ObservedPeer,
1929    ) {
1930        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1931        let peer = match peer {
1932            OsPeer::Unobserved => ObservedPeer::unobserved(),
1933            OsPeer::Observed(peer) => {
1934                let observed = match peer.user {
1935                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1936                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1937                };
1938                observed
1939                    .with_exe(
1940                        peer.exe
1941                            .and_then(|path| path.into_os_string().into_string().ok()),
1942                    )
1943                    .with_signing_id(peer.signing_id)
1944            }
1945        };
1946        self.observed_peers
1947            .lock()
1948            .unwrap_or_else(|p| p.into_inner())
1949            .insert(connection, peer);
1950    }
1951
1952    pub(crate) fn record_caller_endpoints(
1953        &self,
1954        connection: ConnectionId,
1955        endpoints: Option<(SocketAddr, SocketAddr)>,
1956    ) {
1957        let mut callers = self
1958            .caller_endpoints
1959            .lock()
1960            .unwrap_or_else(|p| p.into_inner());
1961        callers.remove(&connection);
1962        if let Some(endpoints) = endpoints {
1963            callers.insert(connection, endpoints);
1964        }
1965    }
1966
1967    #[cfg(test)]
1968    pub(crate) fn with_caller_lookup_for_test(
1969        mut self,
1970        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1971    ) -> Self {
1972        self.caller_lookup = Some(Arc::new(lookup));
1973        self
1974    }
1975
1976    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1977        let endpoints = self
1978            .caller_endpoints
1979            .lock()
1980            .unwrap_or_else(|p| p.into_inner())
1981            .get(&connection)
1982            .copied();
1983        #[cfg(test)]
1984        let lookup = self.caller_lookup.clone();
1985        tokio::task::spawn_blocking(move || {
1986            #[cfg(test)]
1987            if let Some(lookup) = lookup {
1988                return lookup(endpoints);
1989            }
1990            endpoints
1991                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1992                .unwrap_or_default()
1993        })
1994        .await
1995        .unwrap_or_default()
1996    }
1997
1998    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1999        frame.header.channel == 0
2000            && frame.header.ty == FrameType::Request
2001            && self
2002                .forwarding
2003                .module_endpoint_for_connection(ctx.connection_id)
2004                .is_ok_and(|endpoint| endpoint.is_none())
2005            && parse_client_control_request(&frame.body)
2006                .ok()
2007                .is_some_and(|request| privileged_operation(&request).is_some())
2008    }
2009
2010    pub(crate) async fn handle_control_frame_timed(
2011        &self,
2012        ctx: &RouteCtx,
2013        frame: Frame,
2014        dispatch_started_at: Option<StdInstant>,
2015    ) -> Result<Vec<Frame>, RouterError> {
2016        match frame.header.ty {
2017            FrameType::Ping => Ok(vec![pong(&frame)?]),
2018            FrameType::Hello => {
2019                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2020            }
2021            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2022            FrameType::Cancel => {
2023                // A Cancel on channel 0 names a waiting operator.confirm or a
2024                // subscription. Successful cancellation has no ordinary reply.
2025                if self
2026                    .forwarding
2027                    .operator_confirms()
2028                    .cancel(ctx.connection_id, frame.header.corr)
2029                    || self
2030                        .supervisor
2031                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2032                    || self.cancel_activity_subscription(ctx, &frame)?
2033                {
2034                    Ok(Vec::new())
2035                } else {
2036                    Ok(vec![control_error_frame(
2037                        &frame,
2038                        "unknown_subscription",
2039                        "no subscription has this correlation id",
2040                    )?])
2041                }
2042            }
2043            FrameType::Request => {
2044                // This additive operation is not part of the existing exhaustive
2045                // module-control enum. Probe the op before decoding that enum.
2046                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2047                if op
2048                    .as_ref()
2049                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2050                {
2051                    return self.handle_activity_record(ctx, frame).await;
2052                }
2053                if op.as_ref().is_some_and(|probe| {
2054                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2055                }) {
2056                    return self.handle_activity_read(ctx, frame).await;
2057                }
2058                if op
2059                    .as_ref()
2060                    .is_some_and(|probe| probe.op == "operator.confirm")
2061                {
2062                    return self.handle_operator_confirm(ctx, frame);
2063                }
2064                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2065                    return self
2066                        .observe_control_wait(
2067                            ctx,
2068                            None,
2069                            ops::APP_OPEN,
2070                            self.handle_app_open(ctx, frame),
2071                        )
2072                        .await;
2073                }
2074                if op
2075                    .as_ref()
2076                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2077                {
2078                    return self.handle_connection_describe(ctx, frame);
2079                }
2080                if op
2081                    .as_ref()
2082                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2083                {
2084                    return self.handle_scope_served_by(ctx.connection_id, frame);
2085                }
2086                if self
2087                    .forwarding
2088                    .module_endpoint_for_connection(ctx.connection_id)
2089                    .map_err(RouterError::Forwarding)?
2090                    .is_some()
2091                {
2092                    if !is_known_module_request_op(&frame.body) {
2093                        return Ok(vec![control_error_frame(
2094                            &frame,
2095                            "unsupported_control_frame",
2096                            "module-originated channel-0 REQUEST is not supported",
2097                        )?]);
2098                    }
2099                    let request = match parse_module_control_request_from_module(&frame.body) {
2100                        Ok(request) => request,
2101                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2102                            return Ok(vec![control_error_frame(
2103                                &frame,
2104                                "unsupported_control_frame",
2105                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2106                            )?])
2107                        }
2108                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2109                            return Ok(vec![control_error_frame(
2110                                &frame,
2111                                "invalid_control_body",
2112                                format!("malformed module control body: {err}"),
2113                            )?])
2114                        }
2115                    };
2116                    let op = module_control_request_op(&request);
2117                    let corr = frame.header.corr;
2118                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2119                    let result =
2120                        self.handle_module_control_request(ctx.connection_id, frame, request);
2121                    log_slow_control_dispatch(
2122                        dispatch_started_at,
2123                        op,
2124                        ctx.connection_id,
2125                        corr,
2126                        None,
2127                    );
2128                    return result;
2129                }
2130
2131                if is_known_module_request_op(&frame.body) {
2132                    return Ok(vec![control_error_frame(
2133                        &frame,
2134                        "not_registered",
2135                        "catalog.update requires an active module registration owned by this connection",
2136                    )?]);
2137                }
2138
2139                let request = match parse_client_control_request(&frame.body) {
2140                    Ok(request) => request,
2141                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2142                        return Ok(vec![control_error_frame(
2143                            &frame,
2144                            "unknown_control_op",
2145                            format!("unknown client control op: {err}"),
2146                        )?])
2147                    }
2148                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2149                        return Ok(vec![control_error_frame(
2150                            &frame,
2151                            "invalid_control_body",
2152                            format!("malformed client control body: {err}"),
2153                        )?])
2154                    }
2155                };
2156                let op = client_control_request_op(&request);
2157                let corr = frame.header.corr;
2158                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2159                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2160                #[cfg(test)]
2161                if let Some(delay) = self.control_dispatch_delay {
2162                    tokio::time::sleep(delay).await;
2163                }
2164                let result = self
2165                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2166                    .await;
2167                if let Some(timings) = route_open_timings.as_mut() {
2168                    timings.finish();
2169                }
2170                log_slow_control_dispatch(
2171                    dispatch_started_at,
2172                    op,
2173                    ctx.connection_id,
2174                    corr,
2175                    route_open_timings.as_ref(),
2176                );
2177                result
2178            }
2179            FrameType::Push => {
2180                let Some(endpoint) = self
2181                    .forwarding
2182                    .module_endpoint_for_connection(ctx.connection_id)
2183                    .map_err(RouterError::Forwarding)?
2184                else {
2185                    return Ok(vec![control_error_frame(
2186                        &frame,
2187                        "unsupported_control_frame",
2188                        "client-originated channel-0 PUSH is not supported",
2189                    )?]);
2190                };
2191                self.handle_status_update(endpoint, frame)
2192            }
2193            FrameType::Response | FrameType::Error
2194                if self
2195                    .forwarding
2196                    .module_endpoint_for_connection(ctx.connection_id)
2197                    .map_err(RouterError::Forwarding)?
2198                    .is_some() =>
2199            {
2200                self.handle_module_relay_response(ctx.connection_id, frame)
2201            }
2202            ty => Ok(vec![control_error_frame(
2203                &frame,
2204                "unsupported_control_frame",
2205                format!("unsupported channel-0 frame {ty:?}"),
2206            )?]),
2207        }
2208    }
2209
2210    pub fn cleanup_connection(
2211        &self,
2212        connection_id: ConnectionId,
2213    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2214        self.local_connections
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner())
2217            .remove(&connection_id);
2218        self.caller_endpoints
2219            .lock()
2220            .unwrap_or_else(|p| p.into_inner())
2221            .remove(&connection_id);
2222        self.build_class_warned
2223            .lock()
2224            .unwrap_or_else(|p| p.into_inner())
2225            .remove(&connection_id);
2226        self.observed_peers
2227            .lock()
2228            .unwrap_or_else(|p| p.into_inner())
2229            .remove(&connection_id);
2230        self.operator_apps
2231            .lock()
2232            .unwrap_or_else(|p| p.into_inner())
2233            .remove(&connection_id);
2234        self.forwarding
2235            .operator_confirms()
2236            .connection_closed(connection_id);
2237        self.cleanup_connection_with_end_reason(
2238            connection_id,
2239            RegistrationEndReason::ConnectionClosed,
2240        )
2241    }
2242
2243    fn cleanup_connection_with_end_reason(
2244        &self,
2245        connection_id: ConnectionId,
2246        requested_reason: RegistrationEndReason,
2247    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2248        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2249            self.registry
2250                .get_module_by_connection(connection_id)?
2251                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2252                .and_then(|module| module.registration_end_reason().ok().flatten())
2253                .unwrap_or(requested_reason)
2254        } else {
2255            requested_reason
2256        };
2257        let crash_closed = self
2258            .registry
2259            .get_module_by_connection(connection_id)?
2260            .and_then(|registration| {
2261                self.forwarding
2262                    .module_endpoint_for_connection(connection_id)
2263                    .ok()
2264                    .flatten()
2265                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2266                    .map(|routes| (registration.manifest.module_id, routes))
2267            });
2268        let crash_closed = crash_closed.map(|(module_id, routes)| {
2269            let terminal = match self.supervisor.get(&module_id) {
2270                None => false,
2271                Some(module) => match module.will_recover_after_connection_loss() {
2272                    Ok(will_recover) => !will_recover,
2273                    Err(err) => {
2274                        warn!(
2275                            %module_id,
2276                            error = %err,
2277                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2278                        );
2279                        false
2280                    }
2281                },
2282            };
2283            // The forwarding table gates all providers at the start of daemon
2284            // shutdown, before their connections are closed. An ordinary
2285            // module disconnect still reports crash if that gate is not set.
2286            let (reason, drained) = match self
2287                .forwarding
2288                .daemon_drain_result_for_connection(connection_id)
2289            {
2290                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2291                Ok(None) => (RouteCloseReason::Crash, false),
2292                Err(err) => {
2293                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2294                    (RouteCloseReason::Crash, false)
2295                }
2296            };
2297            (module_id, routes, reason, drained, terminal)
2298        });
2299        let registrations = self.deregister_connection(connection_id, end_reason);
2300        let cleanup = if crash_closed.is_some() {
2301            self.forwarding.cleanup_connection_counted(connection_id)
2302        } else {
2303            // A module connection's teardown needs the count of abandoned
2304            // route.bind relays for its route.closed notice below. Any other
2305            // connection, such as a client's, sends no such notice and needs
2306            // only its routes released, so it uses the route-only wrapper and
2307            // reports zero.
2308            self.forwarding
2309                .cleanup_connection(connection_id)
2310                .map(|released| crate::forwarding::ConnectionCleanup {
2311                    released,
2312                    abandoned_relays: 0,
2313                })
2314        };
2315        // The route.closed push waits for forwarding teardown because only
2316        // teardown knows how many pending route.bind relays it aborted. It still
2317        // goes out before the GOODBYEs for the released routes, and its targets
2318        // were captured above, before teardown removed those routes.
2319        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2320            let abandoned = cleanup
2321                .as_ref()
2322                .map_or(0, |cleanup| cleanup.abandoned_relays);
2323            send_route_control_pushes(
2324                &self.forwarding,
2325                routes,
2326                ClientControlPush::RouteClosed {
2327                    module_id,
2328                    channels: Vec::new(),
2329                    reason,
2330                    drained,
2331                    abandoned,
2332                    excluded_subscriptions: 0,
2333                    terminal: Some(terminal),
2334                },
2335            );
2336        }
2337        if let Ok(cleanup) = cleanup {
2338            self.emit_route_goodbyes(cleanup.released);
2339        }
2340        // Signal the registration-release watch only now that BOTH registry and
2341        // forwarding teardown are done, so a supervisor waiting to spawn a
2342        // replacement never observes release while old routes still exist.
2343        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2344            crate::supervise::notify_registration_release();
2345            self.capability_evaluator.wake_deadline_loop();
2346            self.refresh_capability_requirements();
2347        }
2348        self.supervisor.remove_spawn_subscribers(connection_id);
2349        self.activity_streams
2350            .lock()
2351            .unwrap_or_else(|p| p.into_inner())
2352            .retain(|(id, _), task| {
2353                if *id == connection_id {
2354                    task.abort();
2355                    false
2356                } else {
2357                    true
2358                }
2359            });
2360        // Sync authority dies with its connection, so the owner's next
2361        // connection can take it; the owner's scopes stay as they are.
2362        self.hello_launch_nonces
2363            .lock()
2364            .unwrap_or_else(|poisoned| poisoned.into_inner())
2365            .forget(connection_id);
2366        self.scopes
2367            .write()
2368            .unwrap_or_else(|poisoned| poisoned.into_inner())
2369            .release_connection(connection_id);
2370        registrations
2371    }
2372
2373    pub(crate) fn handle_route_goodbye(
2374        &self,
2375        connection_id: ConnectionId,
2376        route_channel: u16,
2377        route_epoch: u32,
2378    ) -> Result<bool, RouterError> {
2379        debug!(
2380            connection_id = connection_id.get(),
2381            route_channel, route_epoch, "handling route GOODBYE"
2382        );
2383        let RouteRelease::Removed(released_route) = self
2384            .forwarding
2385            .release_client_route(connection_id, route_channel, route_epoch)
2386            .map_err(RouterError::Forwarding)?
2387        else {
2388            return Ok(false);
2389        };
2390        self.emit_route_goodbyes(vec![released_route]);
2391        Ok(true)
2392    }
2393
2394    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2395        for released in released_routes {
2396            let frame = match Frame::build_with_version(
2397                released.negotiated_ver,
2398                FrameType::Goodbye,
2399                control_flags(),
2400                released.channel,
2401                released.epoch,
2402                0,
2403                Vec::new(),
2404            ) {
2405                Ok(frame) => frame,
2406                Err(err) => {
2407                    warn!(
2408                        route_channel = released.channel,
2409                        error = %err,
2410                        "failed to build route GOODBYE frame"
2411                    );
2412                    continue;
2413                }
2414            };
2415            if !released.close_on_delivery_failure() {
2416                crate::forwarding::send_module_route_goodbye(
2417                    &self.counters,
2418                    &released.sink,
2419                    frame,
2420                    released.module_id.as_deref(),
2421                    "client route released",
2422                );
2423                continue;
2424            }
2425            if let Err(err) = released.sink.try_send(frame) {
2426                warn!(
2427                    target_connection_id = released.connection_id.get(),
2428                    route_channel = released.channel,
2429                    error = %err,
2430                    "route GOODBYE was not delivered to client; closing target connection"
2431                );
2432                if self
2433                    .forwarding
2434                    .escalate_client_delivery_failure(
2435                        released.connection_id,
2436                        released.channel,
2437                        released.epoch,
2438                        CloseReason::new(
2439                            "route_goodbye_delivery_failed",
2440                            format!(
2441                                "failed to enqueue route GOODBYE for channel {}: {err}",
2442                                released.channel
2443                            ),
2444                        ),
2445                        crate::forwarding::UndeliveredFrame {
2446                            module_id: released.module_id.as_deref(),
2447                            sink: &released.sink,
2448                        },
2449                    )
2450                    .unwrap_or(false)
2451                {
2452                    self.counters.increment_goodbye_relay_client_failed();
2453                }
2454            }
2455        }
2456    }
2457
2458    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2459    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2460    /// own commit failed after the module had already accepted). Without this, a
2461    /// module that accepts late keeps a binding subc has torn down, so a later
2462    /// frame on that module channel could misdeliver if the channel is reused.
2463    ///
2464    /// Never closes the shared module connection on failure: a dropped notification
2465    /// only wastes a bounded amount of warm module-side state, which the module's
2466    /// own idle reaper reclaims. Only call this once the route.bind relay was
2467    /// actually enqueued to the module — if the relay send itself failed, the
2468    /// module never created a binding and there is nothing to tear down.
2469    fn send_abandoned_route_bind_goodbye(
2470        &self,
2471        module_sink: &crate::FrameSink,
2472        negotiated_ver: u8,
2473        module_channel: u16,
2474        module_epoch: u32,
2475    ) {
2476        let frame = match Frame::build_with_version(
2477            negotiated_ver,
2478            FrameType::Goodbye,
2479            control_flags(),
2480            module_channel,
2481            module_epoch,
2482            0,
2483            Vec::new(),
2484        ) {
2485            Ok(frame) => frame,
2486            Err(err) => {
2487                warn!(
2488                    route_channel = module_channel,
2489                    error = %err,
2490                    "failed to build GOODBYE for abandoned route.bind"
2491                );
2492                return;
2493            }
2494        };
2495        crate::forwarding::send_module_route_goodbye(
2496            &self.counters,
2497            module_sink,
2498            frame,
2499            None,
2500            "abandoned route.bind",
2501        );
2502    }
2503
2504    fn handle_hello(
2505        &self,
2506        connection_id: ConnectionId,
2507        sink: Option<crate::FrameSink>,
2508        frame: Frame,
2509    ) -> Result<Vec<Frame>, RouterError> {
2510        debug!(
2511            connection_id = connection_id.get(),
2512            corr = frame.header.corr,
2513            "handling HELLO"
2514        );
2515        // A module connection has one identity for its entire lifetime. A second
2516        // registration would leave the old registry owner behind while replacing
2517        // its forwarding endpoint and launch nonce.
2518        if self
2519            .registry
2520            .get_module_by_connection(connection_id)
2521            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2522            .is_some()
2523        {
2524            return Ok(vec![control_error_frame(
2525                &frame,
2526                "invalid_hello",
2527                "connection is already registered as a module",
2528            )?]);
2529        }
2530        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2531            Ok(value) => value,
2532            Err(err) => {
2533                return Ok(vec![control_error_frame(
2534                    &frame,
2535                    "invalid_hello",
2536                    format!("malformed HELLO body: {err}"),
2537                )?])
2538            }
2539        };
2540        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2541            return Ok(vec![control_error_frame(
2542                &frame,
2543                "invalid_capability_grammar",
2544                err.to_string(),
2545            )?]);
2546        }
2547        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2548            return Ok(vec![control_error_frame(
2549                &frame,
2550                "invalid_manifest",
2551                err.to_string(),
2552            )?]);
2553        }
2554        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2555            return Ok(vec![control_error_frame(
2556                &frame,
2557                "invalid_event_declaration",
2558                err.to_string(),
2559            )?]);
2560        }
2561        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2562            return Ok(vec![control_error_frame(
2563                &frame,
2564                "invalid_manifest",
2565                err.to_string(),
2566            )?]);
2567        }
2568        if let Some(provenance) = hello_value
2569            .get("manifest")
2570            .and_then(|manifest| manifest.get("provenance"))
2571        {
2572            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2573                return Ok(vec![control_error_frame(
2574                    &frame,
2575                    "invalid_manifest",
2576                    format!("malformed manifest provenance: {err}"),
2577                )?]);
2578            }
2579        }
2580        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2581            let module_id = hello_value["manifest"]["module_id"]
2582                .as_str()
2583                .unwrap_or("<unknown>");
2584            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2585        }
2586        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2587            let module_id = hello_value["manifest"]["module_id"]
2588                .as_str()
2589                .unwrap_or("<unknown>");
2590            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2591        }
2592        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2593            let module_id = hello_value["manifest"]["module_id"]
2594                .as_str()
2595                .unwrap_or("<unknown>");
2596            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2597        }
2598        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2599            Ok(hello) => hello,
2600            Err(err) => {
2601                return Ok(vec![control_error_frame(
2602                    &frame,
2603                    "invalid_hello",
2604                    format!("malformed HELLO body: {err}"),
2605                )?])
2606            }
2607        };
2608
2609        if hello.protocol_ver != hello.manifest.protocol_ver {
2610            return Ok(vec![control_error_frame(
2611                &frame,
2612                "invalid_manifest",
2613                format!(
2614                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2615                    hello.protocol_ver, hello.manifest.protocol_ver
2616                ),
2617            )?]);
2618        }
2619
2620        if hello.manifest.module_id.trim().is_empty() {
2621            return Ok(vec![control_error_frame(
2622                &frame,
2623                "invalid_manifest",
2624                "manifest module_id must not be empty",
2625            )?]);
2626        }
2627
2628        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2629            Ok(negotiated_ver) => negotiated_ver,
2630            Err(message) => {
2631                return Ok(vec![control_error_frame(
2632                    &frame,
2633                    "version_unsupported",
2634                    message,
2635                )?])
2636            }
2637        };
2638
2639        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2640        // swap is open for this id, the only HELLO admitted as a second process
2641        // is the one carrying the candidate's launch nonce (the swap token), and
2642        // it registers into the candidate slot rather than being refused as a
2643        // duplicate. Run after the reserved gate, a reserved module's candidate
2644        // would be refused `reserved_module` for presenting a nonce that gate
2645        // does not know. See `SupervisorHandle::swap_hello_admission`.
2646        let swap_admission = self
2647            .supervisor
2648            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2649        if swap_admission == SwapHelloAdmission::Refused {
2650            warn!(
2651                module_id = %hello.manifest.module_id,
2652                connection_id = connection_id.get(),
2653                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2654            );
2655            return Ok(vec![control_error_frame(
2656                &frame,
2657                "swap_token_invalid",
2658                format!(
2659                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2660                    hello.manifest.module_id
2661                ),
2662            )?]);
2663        }
2664        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2665
2666        // A swap candidate proves its identity with the candidate nonce above.
2667        // Other registrations receive this catalog fact only when reserved-module
2668        // policy required and accepted a nonce; unreserved registrations require
2669        // no nonce check.
2670        let nonce_verified = if swap_candidate {
2671            true
2672        } else {
2673            match self.supervisor.reserved_hello_verification(
2674                &hello.manifest.module_id,
2675                hello.launch_nonce.as_deref(),
2676            ) {
2677                Ok(verified) => verified,
2678                Err(rejection) => {
2679                    let message = match rejection {
2680                        ReservedHelloRejection::Exact { module_id } => format!(
2681                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2682                        ),
2683                        ReservedHelloRejection::Prefix {
2684                            prefix,
2685                            owner_module_id,
2686                        } => format!(
2687                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2688                            hello.manifest.module_id
2689                        ),
2690                    };
2691                    return Ok(vec![control_error_frame(
2692                        &frame,
2693                        "reserved_module",
2694                        message,
2695                    )?]);
2696                }
2697            }
2698        };
2699
2700        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2701            &hello.manifest.module_id,
2702            hello.manifest.capabilities.as_ref(),
2703        );
2704        if let Some(refusal) = reserved_capability_refusals.first() {
2705            let capability = refusal.capability.clone();
2706            let bound_module = refusal.claimants[0].clone();
2707            log_duplicate_claim_events(reserved_capability_refusals);
2708            return Ok(vec![control_error_frame(
2709                &frame,
2710                "reserved_capability",
2711                format!(
2712                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2713                    capability, bound_module, hello.manifest.module_id
2714                ),
2715            )?]);
2716        }
2717
2718        // A connection that already opened client routes must not also register as
2719        // a module: cleanup would then release only one side and leak the other.
2720        if self
2721            .forwarding
2722            .connection_has_client_routes(connection_id)
2723            .map_err(RouterError::Forwarding)?
2724        {
2725            return Ok(vec![control_error_frame(
2726                &frame,
2727                "invalid_hello",
2728                "connection has open client routes and cannot also register as a module",
2729            )?]);
2730        }
2731
2732        // Kept for scope sync authority, which goes only to the connection that
2733        // presented the module's current launch nonce. Recorded before the
2734        // registration is attempted: a connection whose registration then fails
2735        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2736        self.hello_launch_nonces
2737            .lock()
2738            .unwrap_or_else(|poisoned| poisoned.into_inner())
2739            .record(connection_id, hello.launch_nonce.as_deref());
2740        let control_ops = effective_module_control_ops(hello.control_ops);
2741        // Built before anything is registered so an encoding failure leaves no
2742        // registry or forwarding state behind.
2743        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2744        if swap_candidate {
2745            return self.register_swap_candidate(
2746                connection_id,
2747                sink,
2748                &frame,
2749                hello.manifest,
2750                negotiated_ver,
2751                control_ops,
2752                hello_ack,
2753            );
2754        }
2755        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2756        // is refused, but `catalog.list` does not show the module and the
2757        // catalog generation does not move until it is published below, once
2758        // its forwarding endpoint exists. Listing it now would advertise a
2759        // module that `route.open` still refuses for having no forwarding
2760        // connection.
2761        let registration = match self
2762            .registry
2763            .register_unpublished_verified_with_control_ops(
2764                hello.manifest,
2765                negotiated_ver,
2766                connection_id,
2767                control_ops,
2768                nonce_verified,
2769            ) {
2770            Ok(registration) => registration,
2771            Err(RegistryError::DuplicateModuleId { module_id }) => {
2772                return Ok(vec![control_error_frame(
2773                    &frame,
2774                    "duplicate_module_id",
2775                    format!(
2776                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2777                    ),
2778                )?])
2779            }
2780            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2781                return Ok(vec![control_error_frame(
2782                    &frame,
2783                    "invalid_module_id",
2784                    err.to_string(),
2785                )?])
2786            }
2787            Err(err) => {
2788                return Ok(vec![control_error_frame(
2789                    &frame,
2790                    "registry_error",
2791                    err.to_string(),
2792                )?])
2793            }
2794        };
2795        #[cfg(test)]
2796        self.hold_between_registry_and_forwarding();
2797
2798        let reply = if let Some(sink) = sink {
2799            // The forwarding table's module store is also the daemon-to-module
2800            // control-RPC lane, so every HELLO gets a live endpoint even when the
2801            // manifest has no routable provider role. Non-routable modules still
2802            // cannot receive route.bind in production: `handle_route_open` checks
2803            // the registry manifest with `target_has_required_role` before the
2804            // only production call to `begin_route_bind_relay_for` below that
2805            // route.open path. The remaining direct relay callers are unit tests
2806            // and benchmark harnesses that construct forwarding state explicitly.
2807            //
2808            // The HELLO_ACK is queued by the forwarding table itself, before the
2809            // endpoint becomes visible, and is NOT returned as a reply. A module
2810            // reads HELLO_ACK first and exits on anything else; a reply is only
2811            // written after this handler returns, by which time a route.open on
2812            // another connection could already have queued a route.bind request
2813            // for this module ahead of it.
2814            let concurrency = manifest_concurrency(&registration.manifest);
2815            if let Err(err) = self.forwarding.register_module_connection_acked(
2816                connection_id,
2817                registration.manifest.module_id.clone(),
2818                negotiated_ver,
2819                concurrency,
2820                sink,
2821                hello_ack,
2822            ) {
2823                // Forwarding registration failed, so there is no forwarding
2824                // state to tear down. Remove the registry entry, which was never
2825                // published and so never listed or counted in the catalog
2826                // generation, and signal the release watch directly.
2827                if matches!(
2828                    self.deregister_connection(
2829                        connection_id,
2830                        RegistrationEndReason::RegistrationFailed,
2831                    ),
2832                    Ok(r) if !r.is_empty()
2833                ) {
2834                    crate::supervise::notify_registration_release();
2835                }
2836                return Ok(vec![control_error_frame(
2837                    &frame,
2838                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2839                        "invalid_hello"
2840                    } else {
2841                        forwarding_error_code(&err)
2842                    },
2843                    err.to_string(),
2844                )?]);
2845            }
2846            Vec::new()
2847        } else {
2848            // No sink means no forwarding endpoint, so nothing can be routed
2849            // ahead of the ack; it goes out as the reply.
2850            vec![hello_ack]
2851        };
2852
2853        // The module is routable now (or, without a sink, as routable as it
2854        // will ever be), so the catalog may list it. Publishing bumps the
2855        // catalog generation in the same registry critical section that makes
2856        // the module listed.
2857        if !self
2858            .registry
2859            .publish_registration(connection_id)
2860            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2861        {
2862            debug!(
2863                module_id = %registration.manifest.module_id,
2864                connection_id = connection_id.get(),
2865                "registration was removed before it could be published"
2866            );
2867        }
2868
2869        // Exposure over assumption: Concurrency's serde default is pinned to the
2870        // pre-field behavior (ModuleManaged), so a management surface that is
2871        // genuinely Serial and just never declared it inherits concurrent
2872        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2873        // turns "no module has been bitten yet" into the checkable claim "no
2874        // module is exposed" -- one read of the boot log instead of a fleet
2875        // audit. Detected from the raw HELLO bytes because the serde default
2876        // deliberately erases the absent/declared distinction from the type.
2877        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2878            info!(
2879                module_id = %registration.manifest.module_id,
2880                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2881            );
2882        }
2883
2884        self.apply_registration_capabilities(&registration);
2885
2886        self.supervisor
2887            .record_activity_registration(&registration.manifest.module_id);
2888
2889        info!(
2890            module_id = %registration.manifest.module_id,
2891            module_version = %registration.manifest.module_version,
2892            negotiated_ver,
2893            routable_provider = manifest_provides_routable_role(&registration.manifest),
2894            connection_id = connection_id.get(),
2895            "module registered"
2896        );
2897
2898        Ok(reply)
2899    }
2900
2901    /// Register a HELLO the swap gate admitted into the candidate slot of the
2902    /// registry and of forwarding, where it is reachable over its own
2903    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2904    /// nothing routes to it until the supervisor cuts over.
2905    ///
2906    /// Registry first (unpublished), then forwarding, then publication, the
2907    /// same order as an ordinary HELLO; a forwarding failure removes the
2908    /// registry entry again. The swap supervisor treats a published, ready
2909    /// candidate as registered and promotes it, and promotion needs the
2910    /// candidate's forwarding endpoint, so the candidate is published only
2911    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2912    /// recompute capability verdicts or run the capability deny census
2913    /// (`enforce_capability_denies`): both are about the module in the active
2914    /// slot, which `route.open` can target. A candidate sits in the candidate
2915    /// slot, so it neither provides capabilities to other modules nor can be
2916    /// routed to until promotion moves it into the active slot.
2917    #[allow(clippy::too_many_arguments)]
2918    fn register_swap_candidate(
2919        &self,
2920        connection_id: ConnectionId,
2921        sink: Option<crate::FrameSink>,
2922        frame: &Frame,
2923        manifest: ModuleManifest,
2924        negotiated_ver: u8,
2925        control_ops: Vec<String>,
2926        hello_ack: Frame,
2927    ) -> Result<Vec<Frame>, RouterError> {
2928        let module_id = manifest.module_id.clone();
2929        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2930            manifest,
2931            negotiated_ver,
2932            connection_id,
2933            control_ops,
2934        ) {
2935            Ok(registration) => registration,
2936            Err(RegistryError::DuplicateModuleId { module_id }) => {
2937                return Ok(vec![control_error_frame(
2938                    frame,
2939                    "duplicate_module_id",
2940                    format!(
2941                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2942                    ),
2943                )?])
2944            }
2945            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2946                return Ok(vec![control_error_frame(
2947                    frame,
2948                    "invalid_module_id",
2949                    err.to_string(),
2950                )?])
2951            }
2952            Err(err) => {
2953                return Ok(vec![control_error_frame(
2954                    frame,
2955                    "registry_error",
2956                    err.to_string(),
2957                )?])
2958            }
2959        };
2960        #[cfg(test)]
2961        self.hold_between_registry_and_forwarding();
2962        let reply = if let Some(sink) = sink {
2963            // Same ordering as an ordinary HELLO: the forwarding table queues
2964            // the HELLO_ACK before the candidate endpoint is inserted, because
2965            // a module exits if its first frame after HELLO is anything else.
2966            let concurrency = manifest_concurrency(&registration.manifest);
2967            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2968                connection_id,
2969                module_id.clone(),
2970                negotiated_ver,
2971                concurrency,
2972                sink,
2973                hello_ack,
2974            ) {
2975                if matches!(
2976                    self.deregister_connection(
2977                        connection_id,
2978                        RegistrationEndReason::RegistrationFailed,
2979                    ),
2980                    Ok(r) if !r.is_empty()
2981                ) {
2982                    crate::supervise::notify_registration_release();
2983                }
2984                return Ok(vec![control_error_frame(
2985                    frame,
2986                    forwarding_error_code(&err),
2987                    err.to_string(),
2988                )?]);
2989            }
2990            Vec::new()
2991        } else {
2992            vec![hello_ack]
2993        };
2994        if !self
2995            .registry
2996            .publish_registration(connection_id)
2997            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998        {
2999            debug!(
3000                module_id = %module_id,
3001                connection_id = connection_id.get(),
3002                "swap candidate registration was removed before it could be published"
3003            );
3004        }
3005        self.supervisor.mark_swap_candidate_admitted(&module_id);
3006        info!(
3007            module_id = %module_id,
3008            module_version = %registration.manifest.module_version,
3009            negotiated_ver,
3010            ready = registration.ready,
3011            connection_id = connection_id.get(),
3012            "swap candidate registered; not routable until cutover"
3013        );
3014        Ok(reply)
3015    }
3016
3017    fn build_hello_ack(
3018        &self,
3019        frame: &Frame,
3020        negotiated_ver: u8,
3021        module_id: &str,
3022    ) -> Result<Frame, RouterError> {
3023        let ack = ModuleHelloAckBody {
3024            negotiated_ver,
3025            subc_ops: module_subc_ops(),
3026            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3027            storage: self
3028                .storage_config
3029                .as_ref()
3030                .map(|cfg| cfg.descriptor_for(module_id)),
3031            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3032        };
3033        let body = serde_json::to_vec(&ack).map_err(|err| {
3034            RouterError::backend(
3035                0,
3036                frame.header.corr,
3037                format!("failed to encode HELLO_ACK: {err}"),
3038            )
3039        })?;
3040
3041        Frame::build_with_version(
3042            negotiated_ver,
3043            FrameType::HelloAck,
3044            control_flags(),
3045            0,
3046            0,
3047            frame.header.corr,
3048            body,
3049        )
3050        .map_err(RouterError::FrameBuild)
3051    }
3052
3053    async fn handle_client_control_request(
3054        &self,
3055        ctx: &RouteCtx,
3056        frame: Frame,
3057        request: ClientControlRequest,
3058        route_open_timings: &mut Option<RouteOpenTimings>,
3059    ) -> Result<Vec<Frame>, RouterError> {
3060        if let Some(operation) = privileged_operation(&request) {
3061            let caller = self.resolve_caller(ctx.connection_id).await;
3062            let op = client_control_request_op(&request);
3063            let text = prompt_text(operation, &caller);
3064            match self.privileged_operator_prompt {
3065                PrivilegedOperatorPrompt::Log => {
3066                    info!(target: "subc_daemon::privileged_prompt", operation = op,
3067                        target = ?operation, caller = %caller.description(), summary = %text,
3068                        "privileged request would prompt the operator");
3069                }
3070                PrivilegedOperatorPrompt::Enforce => {
3071                    let key =
3072                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3073                    let stamp = self
3074                        .operator_apps
3075                        .lock()
3076                        .unwrap_or_else(|p| p.into_inner())
3077                        .get(&ctx.connection_id)
3078                        .cloned();
3079                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3080                        &self.operator_app_policy,
3081                        &self.launch_table,
3082                        stamp.as_ref(),
3083                    )
3084                    .then(|| self.shortcut_context(stamp, None, None));
3085                    let outcome = self
3086                        .forwarding
3087                        .operator_confirms()
3088                        .approve_privileged_with_shortcut(
3089                            ctx,
3090                            &frame,
3091                            op,
3092                            text,
3093                            caller.description(),
3094                            key,
3095                            shortcut,
3096                        )
3097                        .await;
3098                    if outcome.code != "confirmed" {
3099                        return Ok(vec![outcome.frame(&frame)]);
3100                    }
3101                }
3102            }
3103        }
3104        match request {
3105            ClientControlRequest::DaemonShutdownIntent { intent } => {
3106                if intent.as_str() != Some("restart") {
3107                    return Ok(vec![control_error_frame(
3108                        &frame,
3109                        "invalid_params",
3110                        "intent must be 'restart'",
3111                    )?]);
3112                }
3113                let response = ClientControlResponse::DaemonShutdownIntent {
3114                    intent: "restart".to_owned(),
3115                    expires_at_ms: self.supervisor.announce_daemon_restart(),
3116                };
3117                Ok(vec![control_response_body_frame(
3118                    &frame,
3119                    &response,
3120                    "ClientControlResponse::DaemonShutdownIntent",
3121                )?])
3122            }
3123            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3124            ClientControlRequest::CatalogList { module_id } => {
3125                self.handle_catalog_list(frame, module_id)
3126            }
3127            ClientControlRequest::RouteOpen {
3128                target,
3129                identity,
3130                consumer_identity,
3131                consumer_capabilities,
3132                role_versions,
3133                busy_refusals,
3134                admission_facts,
3135                scope,
3136                ..
3137            } => {
3138                self.handle_route_open(
3139                    ctx,
3140                    frame,
3141                    RouteOpenRequest {
3142                        target,
3143                        identity,
3144                        consumer_identity,
3145                        consumer_capabilities,
3146                        role_versions,
3147                        busy_refusals,
3148                        admission_facts,
3149                        scope,
3150                    },
3151                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3152                )
3153                .await
3154            }
3155            ClientControlRequest::RoutePoll {
3156                route_channel,
3157                route_epoch,
3158                kind,
3159            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3160            ClientControlRequest::SupervisorList {} => {
3161                // Filesystem-backed diagnostics must not hold up this connection's reader.
3162                let handler = self.clone();
3163                let sink = ctx.egress.clone();
3164                tokio::spawn(
3165                    async move {
3166                        let result = handler.handle_supervisor_list(frame.clone()).await;
3167                        let replies = match result {
3168                            Ok(replies) => replies,
3169                            Err(error) => {
3170                                control_error_frame(&frame, "backend_error", error.to_string())
3171                                    .into_iter()
3172                                    .collect()
3173                            }
3174                        };
3175                        for reply in replies {
3176                            if sink.send(reply).await.is_err() {
3177                                break;
3178                            }
3179                        }
3180                    }
3181                    .with_current_subscriber(),
3182                );
3183                Ok(Vec::new())
3184            }
3185            ClientControlRequest::SupervisorSpawnSnapshot {} => {
3186                self.handle_supervisor_spawn_snapshot(frame)
3187            }
3188            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3189                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3190            }
3191            ClientControlRequest::SupervisorRestart {
3192                module_id,
3193                drain_timeout_ms,
3194            } => {
3195                self.log_supervisor_request_received(
3196                    ctx,
3197                    frame.header.corr,
3198                    ops::SUPERVISOR_RESTART,
3199                    Some(&module_id),
3200                    None,
3201                )?;
3202                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3203                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3204                    .await
3205            }
3206            ClientControlRequest::SupervisorSwap {
3207                module_id,
3208                ready_timeout_ms,
3209            } => {
3210                self.log_supervisor_request_received(
3211                    ctx,
3212                    frame.header.corr,
3213                    ops::SUPERVISOR_SWAP,
3214                    Some(&module_id),
3215                    None,
3216                )?;
3217                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3218                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3219                    .await
3220            }
3221            ClientControlRequest::SupervisorReload { module_id } => {
3222                self.log_supervisor_request_received(
3223                    ctx,
3224                    frame.header.corr,
3225                    ops::SUPERVISOR_RELOAD,
3226                    Some(&module_id),
3227                    None,
3228                )?;
3229                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3230                self.handle_supervisor_reload(frame, module_id, requester)
3231                    .await
3232            }
3233            ClientControlRequest::SupervisorRescan { preview } => {
3234                if !preview {
3235                    self.log_supervisor_request_received(
3236                        ctx,
3237                        frame.header.corr,
3238                        ops::SUPERVISOR_RESCAN,
3239                        None,
3240                        None,
3241                    )?;
3242                }
3243                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3244                let operation = self.handle_supervisor_rescan(
3245                    frame,
3246                    if preview {
3247                        RescanMode::Preview
3248                    } else {
3249                        RescanMode::Execute
3250                    },
3251                    requester,
3252                );
3253                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3254                    .await
3255            }
3256            ClientControlRequest::SupervisorRescanChecked {
3257                preview,
3258                candidate,
3259                expect,
3260            } => {
3261                self.observe_control_wait(
3262                    ctx,
3263                    None,
3264                    ops::SUPERVISOR_RESCAN,
3265                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3266                )
3267                .await
3268            }
3269            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3270                self.log_supervisor_request_received(
3271                    ctx,
3272                    frame.header.corr,
3273                    ops::SUPERVISOR_RELEASE_RESERVED,
3274                    Some(&module_id),
3275                    None,
3276                )?;
3277                self.handle_supervisor_release_reserved(frame, module_id)
3278                    .await
3279            }
3280            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3281                self.log_supervisor_request_received(
3282                    ctx,
3283                    frame.header.corr,
3284                    ops::SUPERVISOR_SET_ENABLED,
3285                    Some(&module_id),
3286                    Some(enabled),
3287                )?;
3288                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3289                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3290                    .await
3291            }
3292            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3293                self.handle_supervisor_health_probe(ctx, frame, module_id)
3294                    .await
3295            }
3296            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3297            ClientControlRequest::SupervisorRoutes { module_id } => {
3298                self.handle_supervisor_routes(frame, module_id)
3299            }
3300            ClientControlRequest::SupervisorProvenance { module_id } => {
3301                self.handle_supervisor_provenance(ctx, frame, module_id)
3302                    .await
3303            }
3304            ClientControlRequest::SupervisorStderrTail {
3305                module_id,
3306                max_lines,
3307                max_bytes,
3308            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3309            ClientControlRequest::SupervisorTerminals { module_id } => {
3310                self.handle_supervisor_terminals(ctx, frame, module_id)
3311                    .await
3312            }
3313        }
3314    }
3315
3316    fn log_supervisor_request_received(
3317        &self,
3318        ctx: &RouteCtx,
3319        corr: u64,
3320        op: &'static str,
3321        module_id: Option<&str>,
3322        enabled: Option<bool>,
3323    ) -> Result<(), RouterError> {
3324        let caller = self.supervisor_requester(ctx, corr)?;
3325        let caller = principal_label(&caller);
3326        // `Option` fields are recorded only when present, so a request with no
3327        // module id (rescan) or no enabled flag simply omits that field.
3328        info!(
3329            target: "control",
3330            op,
3331            module_id,
3332            enabled,
3333            connection_id = ctx.connection_id.get(),
3334            caller = %caller,
3335            "supervisor request received"
3336        );
3337        Ok(())
3338    }
3339
3340    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3341        Ok(self
3342            .registry
3343            .get_module_by_connection(ctx.connection_id)
3344            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3345            .map(|registration| Principal::Reserved {
3346                module_id: registration.manifest.module_id,
3347            })
3348            .unwrap_or(Principal::Direct))
3349    }
3350
3351    fn handle_module_control_request(
3352        &self,
3353        connection_id: ConnectionId,
3354        frame: Frame,
3355        request: ModuleControlRequestFromModule,
3356    ) -> Result<Vec<Frame>, RouterError> {
3357        match request {
3358            ModuleControlRequestFromModule::CatalogUpdate {
3359                provides,
3360                capabilities,
3361                ready,
3362            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3363            ModuleControlRequestFromModule::LiveRoots {} => {
3364                let registered = self
3365                    .registry
3366                    .get_module_by_connection(connection_id)
3367                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3368                let Some(registration) = registered else {
3369                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3370                };
3371                let response = self
3372                    .forwarding
3373                    .live_roots(&registration.manifest.module_id)
3374                    .map_err(RouterError::Forwarding)?;
3375                Ok(vec![control_response_body_frame(
3376                    &frame,
3377                    &response,
3378                    "ModuleControlResponseToModule::LiveRoots",
3379                )?])
3380            }
3381            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3382                self.handle_scope_sync(connection_id, frame, generation, scopes)
3383            }
3384            ModuleControlRequestFromModule::ScopeApply {
3385                generation,
3386                upsert,
3387                end,
3388            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3389            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3390                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3391            }
3392        }
3393    }
3394
3395    fn handle_scope_served_by(
3396        &self,
3397        connection_id: ConnectionId,
3398        frame: Frame,
3399    ) -> Result<Vec<Frame>, RouterError> {
3400        let registered = self
3401            .registry
3402            .get_module_by_connection(connection_id)
3403            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3404        let Some(registered) = registered else {
3405            return Ok(vec![control_error_frame(
3406                &frame,
3407                "not_registered",
3408                "scope.served_by requires an active module registration owned by this connection",
3409            )?]);
3410        };
3411        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3412        {
3413            Ok(request) => request,
3414            Err(err) => {
3415                return Ok(vec![control_error_frame(
3416                    &frame,
3417                    "invalid_control_body",
3418                    format!("malformed scope.served_by body: {err}"),
3419                )?])
3420            }
3421        };
3422        let answer = self
3423            .scopes
3424            .read()
3425            .unwrap_or_else(|poisoned| poisoned.into_inner())
3426            .served_by(
3427                &registered.manifest.module_id,
3428                &request.scope_ref,
3429                request.scope_epoch,
3430                &request.module_id,
3431            );
3432        let reply = subc_control::ScopeServedByReply::new(
3433            answer,
3434            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3435        );
3436        Ok(vec![control_response_body_frame(
3437            &frame,
3438            &reply,
3439            "ScopeServedByReply",
3440        )?])
3441    }
3442
3443    fn handle_operator_confirm(
3444        &self,
3445        ctx: &RouteCtx,
3446        frame: Frame,
3447    ) -> Result<Vec<Frame>, RouterError> {
3448        use crate::operator_confirm::{audit, Outcome};
3449        let registration = self
3450            .registry
3451            .get_module_by_connection(ctx.connection_id)
3452            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3453        let Some(registration) = registration else {
3454            let outcome = Outcome::refusal("not_registered");
3455            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3456            return Ok(vec![outcome.frame(&frame)]);
3457        };
3458        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3459            Ok(request) => request,
3460            Err(_) => {
3461                let outcome = Outcome::refusal("invalid_control_body");
3462                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3463                return Ok(vec![outcome.frame(&frame)]);
3464            }
3465        };
3466        let declared = matches!(
3467            request.kind,
3468            Some(subc_protocol::session::OperatorConfirmKind::Operation)
3469        ) && request.operation.as_ref().is_some_and(|operation| {
3470            registration
3471                .manifest
3472                .confirm_operations
3473                .as_ref()
3474                .is_some_and(|operations| operations.contains(operation))
3475        });
3476        let module_id = registration.manifest.module_id;
3477        // Read the launch nonce (under its own lock) before taking the forwarding
3478        // table's lock below: holding forwarding while waiting on another daemon
3479        // lock risks a lock-order deadlock with paths that take them the other way.
3480        let nonce = self
3481            .hello_launch_nonces
3482            .lock()
3483            .unwrap_or_else(|p| p.into_inner())
3484            .nonce(ctx.connection_id)
3485            .map(str::to_owned);
3486        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3487            self.supervisor
3488                .spawned_consumer_authorized(&module_id, nonce)
3489        });
3490        let confirms = self.forwarding.operator_confirms();
3491        self.forwarding
3492            .with_operator_route(
3493                ctx.connection_id,
3494                request.route_channel,
3495                request.route_epoch,
3496                |binding| {
3497                    let shortcut = binding
3498                        .filter(|binding| {
3499                            nonce_proven
3500                                && crate::operator_confirm::valid_summary(&request.summary)
3501                                && declared
3502                                && crate::operator_confirm::ShortcutContext::route_eligible(
3503                                    &self.operator_app_policy,
3504                                    &self.launch_table,
3505                                    binding,
3506                                )
3507                        })
3508                        .map(|binding| {
3509                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3510                                &self.operator_app_policy,
3511                                binding,
3512                            );
3513                            let stamp = if phone.is_some() {
3514                                None
3515                            } else {
3516                                binding.operator_app.clone()
3517                            };
3518                            self.shortcut_context(
3519                                stamp,
3520                                phone,
3521                                Some((request.route_channel, request.route_epoch)),
3522                            )
3523                        });
3524                    confirms.admit_with_shortcut(
3525                        ctx,
3526                        frame,
3527                        module_id,
3528                        nonce_proven,
3529                        request,
3530                        binding,
3531                        shortcut,
3532                    )
3533                },
3534            )
3535            .map_err(RouterError::Forwarding)
3536    }
3537
3538    fn shortcut_context(
3539        &self,
3540        stamp: Option<crate::operator_apps::OperatorAppStamp>,
3541        phone_fingerprint: Option<String>,
3542        route: Option<(u16, u32)>,
3543    ) -> crate::operator_confirm::ShortcutContext {
3544        crate::operator_confirm::ShortcutContext {
3545            ledger: self.activity.clone(),
3546            forwarding: self.forwarding.clone(),
3547            launches: self.launch_table.clone(),
3548            stamp,
3549            phone_fingerprint,
3550            route,
3551            connections: self.operator_apps.clone(),
3552            #[cfg(test)]
3553            hooks: Default::default(),
3554        }
3555    }
3556
3557    /// Explicit test-only phone policy; daemon configuration cannot select it.
3558    #[cfg(any(test, feature = "test-support"))]
3559    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3560        self.operator_app_policy.phone_enabled = enabled;
3561        self
3562    }
3563
3564    /// Resolve folders before taking the scope-table lock: a slow filesystem
3565    /// must not hold up admissions for unrelated scoped routes.
3566    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3567        let mut scopes = ScopeRecords::from(scopes);
3568        for (index, record) in scopes.records.iter_mut().enumerate() {
3569            if let Some(refusal) = Self::canonicalize_workdir(record) {
3570                scopes.refusals.insert(index, refusal);
3571            }
3572        }
3573        scopes
3574    }
3575
3576    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3577        let workdir = record.workdir.as_mut()?;
3578        let path = std::path::Path::new(workdir);
3579        if !path.is_absolute() {
3580            return Some((
3581                error_codes::INVALID_WORKDIR,
3582                "workdir must be absolute".to_string(),
3583            ));
3584        }
3585        match ProjectRootId::from_path_allowing_missing(path) {
3586            Ok(root) => match root.as_path().to_str() {
3587                Some(canonical) => {
3588                    *workdir = canonical.to_string();
3589                    None
3590                }
3591                None => Some((
3592                    error_codes::INVALID_WORKDIR,
3593                    "canonical workdir is not UTF-8".to_string(),
3594                )),
3595            },
3596            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3597        }
3598    }
3599
3600    /// `scope.sync`: the owner is the module registered on this connection.
3601    /// A connection with no registration (every client connection, `direct`
3602    /// included) is refused `not_registered` before the table is consulted.
3603    fn handle_scope_sync(
3604        &self,
3605        connection_id: ConnectionId,
3606        frame: Frame,
3607        generation: u64,
3608        scopes: Vec<ScopeRecord>,
3609    ) -> Result<Vec<Frame>, RouterError> {
3610        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3611    }
3612
3613    fn handle_scope_change(
3614        &self,
3615        connection_id: ConnectionId,
3616        frame: Frame,
3617        generation: u64,
3618        scopes: Vec<ScopeRecord>,
3619        end: Option<Vec<ScopeEnd>>,
3620    ) -> Result<Vec<Frame>, RouterError> {
3621        let is_apply = end.is_some();
3622        let Some(registration) = self
3623            .registry
3624            .get_module_by_connection(connection_id)
3625            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3626        else {
3627            return Ok(vec![control_error_frame(
3628                &frame,
3629                "not_registered",
3630                "scope.sync and scope.apply require an active module registration owned by this connection",
3631            )?]);
3632        };
3633        let owner = registration.manifest.module_id;
3634        let scopes = Self::prepare_scope_records(scopes);
3635        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3636        let is_current_launch = |connection: ConnectionId| {
3637            self.hello_launch_nonces
3638                .lock()
3639                .unwrap_or_else(|poisoned| poisoned.into_inner())
3640                .presented(connection, current_nonce.as_deref())
3641        };
3642        // Lock order is the scope table, then the forwarding table: the new
3643        // tags are published, and the routes the change closes are selected,
3644        // while the scope table is still write-locked, so no admission can read
3645        // a record whose tag is not yet published.
3646        let mut table = self
3647            .scopes
3648            .write()
3649            .unwrap_or_else(|poisoned| poisoned.into_inner());
3650        let outcome = match end {
3651            Some(end) => table.apply(
3652                &owner,
3653                connection_id,
3654                is_current_launch,
3655                generation,
3656                scopes,
3657                end,
3658            ),
3659            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3660        };
3661        let drained = match &outcome {
3662            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3663            Err(_) => Vec::new(),
3664        };
3665        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3666        drop(table);
3667        if has_deadlines {
3668            self.scope_deadline_added.notify_one();
3669        }
3670        match outcome {
3671            Ok(applied) => {
3672                let counts = ScopeOutcomeCounts::of(&applied.results);
3673                info!(
3674                    owner = %owner,
3675                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3676                    generation,
3677                    records = applied.results.len(),
3678                    created = counts.created,
3679                    replaced = counts.replaced,
3680                    updated = counts.updated,
3681                    unchanged = counts.unchanged,
3682                    refused = counts.refused,
3683                    ended = applied.ended.len(),
3684                    tag_changes = applied.tag_changes.len(),
3685                    routes_closed = drained.len(),
3686                    "scope change accepted"
3687                );
3688                // An accepted scope change can still refuse individual records, and the
3689                // owner is the only party that sees the reply. Name them here so
3690                // an operator can tell a refused session from a missing one
3691                // without the owner's logs. Capped so a call that refuses
3692                // thousands cannot flood the log; the count above is complete.
3693                for refused in applied
3694                    .results
3695                    .iter()
3696                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3697                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3698                {
3699                    warn!(
3700                        owner = %owner,
3701                        generation,
3702                        scope_ref = %refused.scope_ref,
3703                        scope_epoch = refused.scope_epoch,
3704                        code = refused.code.as_deref().unwrap_or(""),
3705                        "scope record refused"
3706                    );
3707                }
3708                self.close_scope_drained_routes(drained);
3709                let response = if is_apply {
3710                    ModuleControlResponseToModule::ScopeApply {
3711                        generation,
3712                        results: applied.results,
3713                        end_results: applied.end_results,
3714                        ended: applied.ended,
3715                    }
3716                } else {
3717                    ModuleControlResponseToModule::ScopeSync {
3718                        generation,
3719                        results: applied.results,
3720                        ended: applied.ended,
3721                    }
3722                };
3723                Ok(vec![control_response_body_frame(
3724                    &frame,
3725                    &response,
3726                    "scope change response",
3727                )?])
3728            }
3729            Err(refusal) => {
3730                info!(
3731                    owner = %owner,
3732                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3733                    generation,
3734                    code = refusal.code,
3735                    "scope change refused"
3736                );
3737                Ok(vec![control_error_frame(
3738                    &frame,
3739                    refusal.code,
3740                    refusal.message,
3741                )?])
3742            }
3743        }
3744    }
3745
3746    /// Tell both ends of each route a scope change closed. The module gets a
3747    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3748    /// the client's route handle. The client also gets `route.closed` with the
3749    /// scope reason, one push per module and reason, so it can tell a revoked
3750    /// route from an ordinary close and not reopen it.
3751    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3752        if drained.is_empty() {
3753            return;
3754        }
3755        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3756            BTreeMap::new();
3757        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3758        for route in drained {
3759            warn!(
3760                module_id = %route.module_id,
3761                reason = ?route.reason,
3762                client_connection_id = route.client.connection_id.get(),
3763                route_channel = route.client.channel,
3764                "closing route because its scope changed"
3765            );
3766            pushes
3767                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3768                .or_insert_with(|| (route.reason, Vec::new()))
3769                .1
3770                .push(EndpointRoute {
3771                    goodbye_target: route.client.clone(),
3772                    principal: Principal::Unverified,
3773                    bound_at: Instant::now(),
3774                    draining: false,
3775                    drain_reason: None,
3776                });
3777            goodbyes.push(route.module);
3778            goodbyes.push(route.client);
3779        }
3780        for ((module_id, _), (reason, routes)) in pushes {
3781            send_route_control_pushes(
3782                &self.forwarding,
3783                routes,
3784                ClientControlPush::RouteClosed {
3785                    module_id,
3786                    channels: Vec::new(),
3787                    reason,
3788                    drained: false,
3789                    abandoned: 0,
3790                    excluded_subscriptions: 0,
3791                    terminal: Some(false),
3792                },
3793            );
3794        }
3795        self.emit_route_goodbyes(goodbyes);
3796    }
3797
3798    /// `scope.describe`: any registered module may read any scope, because a
3799    /// provider must read the scope a route it serves is stamped with.
3800    fn handle_scope_describe(
3801        &self,
3802        connection_id: ConnectionId,
3803        frame: Frame,
3804        owner: Principal,
3805        scope_ref: String,
3806    ) -> Result<Vec<Frame>, RouterError> {
3807        let registered = self
3808            .registry
3809            .get_module_by_connection(connection_id)
3810            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3811        let Some(registered) = registered else {
3812            return Ok(vec![control_error_frame(
3813                &frame,
3814                "not_registered",
3815                "scope.describe requires an active module registration owned by this connection",
3816            )?]);
3817        };
3818        let mut description = self
3819            .scopes
3820            .read()
3821            .unwrap_or_else(|poisoned| poisoned.into_inner())
3822            .describe(&owner, &scope_ref);
3823        // Describe readers also decode strict ScopeAttributes. Attribution is
3824        // safe to omit unless the requesting module declares it understands it.
3825        if !registered
3826            .manifest
3827            .capabilities
3828            .as_ref()
3829            .is_some_and(|caps| {
3830                caps.provides
3831                    .iter()
3832                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3833            })
3834        {
3835            if let Some(stamp) = description.stamp.as_mut() {
3836                stamp.attributes.on_behalf_of_agent = None;
3837            }
3838        }
3839        if !registered
3840            .manifest
3841            .capabilities
3842            .as_ref()
3843            .is_some_and(|caps| {
3844                caps.provides
3845                    .iter()
3846                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3847            })
3848        {
3849            if let Some(stamp) = description.stamp.as_mut() {
3850                stamp.attributes.flow_version = None;
3851                stamp.attributes.flow_code_hash = None;
3852                stamp.attributes.flow_install_id = None;
3853            }
3854        }
3855        let owner_configured = match &owner {
3856            // Ask whether the owner is configured (`is_configured`), not
3857            // whether it is on the roster (`get(..).is_some()`): a supervised
3858            // module's process can register and describe a scope before the
3859            // supervisor has put it on the roster.
3860            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3861            _ => false,
3862        };
3863        let response = ModuleControlResponseToModule::ScopeDescribe {
3864            status: description.status,
3865            scope_epoch: description.scope_epoch,
3866            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3867            owner_synced: description.owner_synced,
3868            owner_configured,
3869            scope: description.stamp,
3870        };
3871        Ok(vec![control_response_body_frame(
3872            &frame,
3873            &response,
3874            "ModuleControlResponseToModule::ScopeDescribe",
3875        )?])
3876    }
3877
3878    fn handle_catalog_update(
3879        &self,
3880        connection_id: ConnectionId,
3881        frame: Frame,
3882        provides: Vec<ProviderRole>,
3883        capabilities: Option<CapabilityDeclarations>,
3884        ready: Option<bool>,
3885    ) -> Result<Vec<Frame>, RouterError> {
3886        self.refresh_capability_requirements();
3887        let Some(registration) = self
3888            .registry
3889            .get_module_by_connection(connection_id)
3890            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3891        else {
3892            return Ok(vec![control_error_frame(
3893                &frame,
3894                "not_registered",
3895                "catalog.update requires an active module registration owned by this connection",
3896            )?]);
3897        };
3898
3899        if let Some(message) =
3900            catalog_update_frozen_field_message(&registration.manifest, &provides)
3901        {
3902            return Ok(vec![control_error_frame(
3903                &frame,
3904                "catalog_update_frozen_field",
3905                message,
3906            )?]);
3907        }
3908
3909        let mut candidate = registration.manifest.clone();
3910        candidate.provides = provides.clone();
3911        candidate.capabilities = capabilities
3912            .clone()
3913            .or_else(|| registration.manifest.capabilities.clone());
3914        if let Some(rich_ui) = &candidate.rich_ui {
3915            if let Err(err) = rich_ui.validate() {
3916                return Ok(vec![control_error_frame(
3917                    &frame,
3918                    "invalid_manifest",
3919                    err.to_string(),
3920                )?]);
3921            }
3922        }
3923        if let Err(err) = candidate.validate_capability_grammar() {
3924            return Ok(vec![control_error_frame(
3925                &frame,
3926                "invalid_capability_grammar",
3927                err.to_string(),
3928            )?]);
3929        }
3930
3931        // Updates must honor the same reserved owner as initial registration;
3932        // otherwise an empty HELLO could acquire the claim after admission.
3933        let mut conflicts = self
3934            .capability_evaluator
3935            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3936        if let Some(conflict) = conflicts.first() {
3937            let message = format!(
3938                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3939                conflict.capability, conflict.claimants[0], candidate.module_id
3940            );
3941            for conflict in &mut conflicts {
3942                conflict.source = DuplicateClaimSource::CatalogUpdate;
3943            }
3944            log_duplicate_claim_events(conflicts);
3945            return Ok(vec![control_error_frame(
3946                &frame,
3947                "reserved_capability",
3948                message,
3949            )?]);
3950        }
3951
3952        let updated = self
3953            .registry
3954            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3955            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3956        if updated.is_none() {
3957            return Ok(vec![control_error_frame(
3958                &frame,
3959                "not_registered",
3960                "catalog.update requires an active module registration owned by this connection",
3961            )?]);
3962        }
3963        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3964            log_duplicate_claim_events(
3965                self.capability_evaluator
3966                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3967            );
3968        }
3969        if capability_census_trigger(
3970            registration.manifest.capabilities.as_ref(),
3971            updated
3972                .as_ref()
3973                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3974        ) {
3975            self.enforce_capability_denies();
3976        }
3977        self.refresh_capability_requirements();
3978
3979        let response = ModuleControlResponseToModule::CatalogUpdate {};
3980        control_response_body_frame(
3981            &frame,
3982            &response,
3983            "ModuleControlResponseToModule::CatalogUpdate",
3984        )
3985        .map(|frame| vec![frame])
3986    }
3987
3988    pub(crate) fn with_activity_ledger_config(
3989        mut self,
3990        run_dir: &Path,
3991        incarnation: String,
3992        config: crate::activity::ActivityConfig,
3993    ) -> Self {
3994        self.activity = Some(crate::activity::ActivityLedger::open_config(
3995            run_dir.join("activity.jsonl"),
3996            incarnation,
3997            config,
3998        ));
3999        self.supervisor.set_activity(self.activity.clone());
4000        self
4001    }
4002
4003    pub(crate) async fn configure_activity(
4004        &self,
4005        config: crate::activity::ActivityConfig,
4006    ) -> Result<(), &'static str> {
4007        match &self.activity {
4008            Some(ledger) => ledger.configure(config).await,
4009            None => Ok(()),
4010        }
4011    }
4012
4013    fn cancel_activity_subscription(
4014        &self,
4015        ctx: &RouteCtx,
4016        frame: &Frame,
4017    ) -> Result<bool, RouterError> {
4018        let task = self
4019            .activity_streams
4020            .lock()
4021            .unwrap_or_else(|p| p.into_inner())
4022            .remove(&(ctx.connection_id, frame.header.corr));
4023        let Some(task) = task else {
4024            return Ok(false);
4025        };
4026        task.abort();
4027        let end = Frame::build_with_version(
4028            response_version(frame),
4029            FrameType::StreamEnd,
4030            control_flags(),
4031            0,
4032            0,
4033            frame.header.corr,
4034            Vec::new(),
4035        )
4036        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4037        let sink = ctx.egress.clone();
4038        tokio::spawn(async move {
4039            let _ = sink.send(end).await;
4040        });
4041        Ok(true)
4042    }
4043
4044    async fn handle_activity_read(
4045        &self,
4046        ctx: &RouteCtx,
4047        frame: Frame,
4048    ) -> Result<Vec<Frame>, RouterError> {
4049        use subc_control::{
4050            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4051            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4052        };
4053        #[derive(Deserialize)]
4054        struct Identity {
4055            #[serde(default)]
4056            consumer_identity: Option<ConsumerIdentity>,
4057        }
4058        // A registered module connection already identifies its caller. Ignore
4059        // consumer_identity there so a module cannot borrow a mirror's identity.
4060        // Only separate consumer connections use the Direct/nonce trust boundary.
4061        let principal = match self
4062            .forwarding
4063            .module_id_for_connection(ctx.connection_id)
4064            .map_err(RouterError::Forwarding)?
4065        {
4066            Some(module_id) => Ok(Principal::Reserved { module_id }),
4067            None => {
4068                let identity: Identity = match serde_json::from_slice(&frame.body) {
4069                    Ok(identity) => identity,
4070                    Err(_) => {
4071                        return Ok(vec![control_error_frame(
4072                            &frame,
4073                            ACTIVITY_MALFORMED,
4074                            "malformed activity read",
4075                        )?])
4076                    }
4077                };
4078                self.route_open_principal(&frame, identity.consumer_identity)?
4079            }
4080        };
4081        let admitted = matches!(principal, Ok(Principal::Direct))
4082            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4083        if !admitted {
4084            return Ok(vec![control_error_frame(
4085                &frame,
4086                ACTIVITY_READ_FORBIDDEN,
4087                "activity reads require an operator or trusted mirror principal",
4088            )?]);
4089        }
4090        let Some(ledger) = &self.activity else {
4091            return Ok(vec![control_error_frame(
4092                &frame,
4093                ACTIVITY_UNAVAILABLE,
4094                "activity ledger unavailable",
4095            )?]);
4096        };
4097        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4098            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4099        {
4100            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4101                Ok(request) => ledger.query(request).await,
4102                Err(_) => Err(ACTIVITY_MALFORMED),
4103            };
4104            return match result {
4105                Ok(reply) => Ok(vec![control_response_body_frame(
4106                    &frame,
4107                    &reply,
4108                    ops::ACTIVITY_QUERY,
4109                )?]),
4110                Err(code) => Ok(vec![control_error_frame(
4111                    &frame,
4112                    code,
4113                    "activity query refused",
4114                )?]),
4115            };
4116        }
4117        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4118            Ok(request) => request,
4119            Err(_) => {
4120                return Ok(vec![control_error_frame(
4121                    &frame,
4122                    ACTIVITY_MALFORMED,
4123                    "malformed activity subscription",
4124                )?])
4125            }
4126        };
4127        let mut subscription = match ledger.subscribe(request.since).await {
4128            Ok(subscription) => subscription,
4129            Err(code) => {
4130                return Ok(vec![control_error_frame(
4131                    &frame,
4132                    code,
4133                    "activity subscription refused",
4134                )?])
4135            }
4136        };
4137        let key = (ctx.connection_id, frame.header.corr);
4138        let sink = ctx.egress.clone();
4139        let mut streams = self
4140            .activity_streams
4141            .lock()
4142            .unwrap_or_else(|p| p.into_inner());
4143        streams.retain(|_, task| !task.is_finished());
4144        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4145            return Ok(vec![control_error_frame(
4146                &frame,
4147                "duplicate_subscription",
4148                "activity subscription correlation id already in use",
4149            )?]);
4150        }
4151        let task = tokio::spawn(async move {
4152            loop {
4153                let entry = if let Some(entry) = subscription.replay.pop_front() {
4154                    Some(entry)
4155                } else {
4156                    subscription.receiver.recv().await
4157                };
4158                let Some(entry) = entry else {
4159                    break;
4160                };
4161                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4162                    return;
4163                };
4164                let Ok(next) = Frame::build_with_version(
4165                    response_version(&frame),
4166                    FrameType::StreamData,
4167                    control_flags(),
4168                    0,
4169                    0,
4170                    frame.header.corr,
4171                    body,
4172                ) else {
4173                    return;
4174                };
4175                if sink.send(next).await.is_err() {
4176                    return;
4177                }
4178            }
4179            // Dropping a full subscriber's sender drains its accepted entries
4180            // before the terminal error, preserving their order on the wire.
4181            if let Some(code) = subscription.terminal_code() {
4182                if let Ok(end) =
4183                    control_error_frame(&frame, code, "activity subscriber fell behind")
4184                {
4185                    let _ = sink.send(end).await;
4186                }
4187            }
4188        });
4189        streams.insert(key, task.abort_handle());
4190        Ok(Vec::new())
4191    }
4192
4193    async fn handle_activity_record(
4194        &self,
4195        ctx: &RouteCtx,
4196        frame: Frame,
4197    ) -> Result<Vec<Frame>, RouterError> {
4198        use subc_control::{
4199            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4200            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4201        };
4202        let result = async {
4203            let module = self
4204                .forwarding
4205                .module_id_for_connection(ctx.connection_id)
4206                .ok()
4207                .flatten()
4208                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4209            let request: ActivityRecordRequest =
4210                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4211            crate::activity::validate(&module, &request)?;
4212            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4213            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4214            let at_ms = std::time::SystemTime::now()
4215                .duration_since(std::time::UNIX_EPOCH)
4216                .unwrap_or_default()
4217                .as_millis() as u64;
4218            ledger.record(module, request, actor, at_ms).await
4219        }
4220        .await;
4221        match result {
4222            Ok(entry) => Ok(vec![control_response_body_frame(
4223                &frame,
4224                &ActivityRecordResponse { seq: entry.seq },
4225                "activity.record",
4226            )?]),
4227            Err(code) => Ok(vec![control_error_frame(
4228                &frame,
4229                code,
4230                "activity recording refused",
4231            )?]),
4232        }
4233    }
4234
4235    fn activity_actor(
4236        &self,
4237        connection: ConnectionId,
4238        module: &str,
4239        on: Option<&subc_control::ActivityOn>,
4240    ) -> Result<subc_control::ActivityActor, &'static str> {
4241        use crate::forwarding::{DataRoute, DataRouteState};
4242        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4243        let principal = |id: &str| Principal::Reserved {
4244            module_id: id.to_owned(),
4245        };
4246        let mut actor = ActivityActor {
4247            principal: principal(module),
4248            scope_owner: None,
4249            agent_id: None,
4250            run_id: None,
4251            flow_id: None,
4252            project_root: None,
4253            workdir: None,
4254            project_id: None,
4255        };
4256        let scope = match on {
4257            None => return Ok(actor),
4258            Some(ActivityOn::Scope(scope)) => {
4259                let selector = subc_protocol::scope::ScopeSelector::new(
4260                    principal(module),
4261                    scope.scope_ref.clone(),
4262                    Some(scope.epoch),
4263                );
4264                self.scopes
4265                    .read()
4266                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4267                    .attribute(&principal(module), module, &selector, true)
4268                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4269            }
4270            Some(ActivityOn::Route(route)) => {
4271                let DataRoute::Module(DataRouteState::Bound(binding)) = self
4272                    .forwarding
4273                    .lookup_data_route(connection, route.channel, route.epoch)
4274                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4275                else {
4276                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4277                };
4278                if binding.module_id != module {
4279                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4280                }
4281                actor.principal = binding.principal.clone();
4282                actor.project_root = binding
4283                    .project_root
4284                    .as_ref()
4285                    .map(|root| root.as_path().to_path_buf());
4286                let Some(bound) = &binding.scope else {
4287                    return Ok(actor);
4288                };
4289                let owner = principal(&bound.owner);
4290                let selector = subc_protocol::scope::ScopeSelector::new(
4291                    owner.clone(),
4292                    bound.scope_ref.clone(),
4293                    Some(bound.tag.scope_epoch),
4294                );
4295                let admitted = self
4296                    .scopes
4297                    .read()
4298                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4299                    .attribute(&owner, module, &selector, true)
4300                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4301                if admitted.tag != bound.tag {
4302                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4303                }
4304                admitted
4305            }
4306        };
4307        actor.scope_owner = Some(scope.stamp.owner);
4308        actor.agent_id = scope.stamp.attributes.agent_id;
4309        actor.run_id = scope.stamp.attributes.run_id;
4310        actor.flow_id = scope.stamp.attributes.flow_id;
4311        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4312        actor.project_id = scope.stamp.project_id;
4313        Ok(actor)
4314    }
4315
4316    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4317        self.refresh_capability_requirements();
4318        // A bare connection count is ambiguous between many clients holding a
4319        // route each and one client accumulating hundreds, so publish the
4320        // concentration alongside it. Route state is best-effort here: a
4321        // diagnostic endpoint must still answer if the forwarding lock is
4322        // contended.
4323        let mut counters = self.counters.snapshot();
4324        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4325            obj.insert(
4326                "activity_unparseable_lines".into(),
4327                ledger.read_errors().into(),
4328            );
4329            obj.insert(
4330                "activity_lifecycle_append_failures".into(),
4331                ledger.lifecycle_failures().into(),
4332            );
4333            obj.insert(
4334                "activity_started_digest_skips".into(),
4335                ledger.lifecycle_digest_skips().into(),
4336            );
4337        }
4338        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4339            self.forwarding.client_route_concentration(),
4340            counters.as_object_mut(),
4341        ) {
4342            obj.insert(
4343                "client_connections_with_routes".into(),
4344                connections_with_routes.into(),
4345            );
4346            obj.insert("max_routes_on_one_connection".into(), max.into());
4347        }
4348        // A module that is being fast-refused and a module that is fine look
4349        // identical from a client that retries and succeeds, so name the open
4350        // breakers here. This rides the existing free-form counters object
4351        // rather than a new wire field, so no sibling that deserializes
4352        // `ServerDescribe` has to be rebuilt to keep reading it.
4353        if let (Some(open_breakers), Some(obj)) = (
4354            self.route_bind_breakers.open_snapshot(),
4355            counters.as_object_mut(),
4356        ) {
4357            obj.insert("route_bind_breakers_open".into(), open_breakers);
4358        }
4359        // A daemon whose local endpoint could not be bound keeps serving TCP,
4360        // and the connection file names no endpoint. Name the reason here, in
4361        // the free-form counters object, so an operator reading daemon facts
4362        // sees it without searching the log, and no wire field changes.
4363        if let (Some(reason), Some(obj)) = (
4364            self.local_endpoint_unavailable.clone(),
4365            counters.as_object_mut(),
4366        ) {
4367            obj.insert("local_endpoint_unavailable".into(), reason.into());
4368        }
4369        let response = ClientControlResponse::ServerDescribe {
4370            protocol_ver: PROTOCOL_VERSION,
4371            subc_ops: subc_ops(),
4372            capabilities: self.describe_capabilities(),
4373            connected_clients: self.connected_clients.count(),
4374            counters: Some(counters),
4375            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4376            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4377            capability_requirements: self.capability_requirement_statuses(),
4378            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4379            config_path: self
4380                .rescan
4381                .as_ref()
4382                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4383        };
4384        Ok(vec![control_response_body_frame(
4385            &frame,
4386            &response,
4387            "ClientControlResponse::ServerDescribe",
4388        )?])
4389    }
4390
4391    fn handle_catalog_list(
4392        &self,
4393        frame: Frame,
4394        module_id: Option<String>,
4395    ) -> Result<Vec<Frame>, RouterError> {
4396        let (generation, modules) = self
4397            .registry
4398            .list_modules_with_nonce_verification()
4399            .map_err(|err| {
4400                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4401            })?;
4402        let entries = modules
4403            .into_iter()
4404            .filter(|(registration, _)| {
4405                module_id
4406                    .as_deref()
4407                    .map(|wanted| registration.manifest.module_id == wanted)
4408                    .unwrap_or(true)
4409            })
4410            .map(|(registration, nonce_verified)| {
4411                let not_ready = self.not_ready_reason(&registration);
4412                let roles = registration.manifest.provides;
4413                CatalogEntry::new(
4414                    registration.manifest.module_id,
4415                    roles,
4416                    registration.control_ops,
4417                )
4418                .with_ready(not_ready.is_none())
4419                .with_not_ready(not_ready)
4420                .with_module_version(Some(registration.manifest.module_version))
4421                .with_provenance(registration.manifest.provenance)
4422                .with_capabilities(registration.manifest.capabilities)
4423                .with_self_signals(registration.manifest.self_signals)
4424                .with_events(registration.manifest.events)
4425                .with_operation_delivery(registration.manifest.operation_delivery)
4426                .with_operation_payload(registration.manifest.operation_payload)
4427                .with_flow_operations(registration.manifest.flow_operations)
4428                .with_confirm_operations(registration.manifest.confirm_operations)
4429                .with_rich_ui(registration.manifest.rich_ui)
4430                .with_role_declarations(registration.manifest.role_declarations)
4431                .with_nonce_verified(nonce_verified)
4432            })
4433            .collect();
4434        let response = ClientControlResponse::CatalogList {
4435            generation,
4436            modules: entries,
4437            subc_ops: subc_ops(),
4438        };
4439        Ok(vec![control_response_body_frame(
4440            &frame,
4441            &response,
4442            "ClientControlResponse::CatalogList",
4443        )?])
4444    }
4445
4446    /// The principal a route.open's `consumer_identity` attests: `Direct`
4447    /// when it presents none, `Reserved` when its launch nonce verifies, and
4448    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4449    /// can run before any refusal check without changing which check refuses.
4450    fn attest_route_open_consumer(
4451        &self,
4452        consumer_identity: Option<&ConsumerIdentity>,
4453    ) -> Option<Principal> {
4454        let Some(consumer_identity) = consumer_identity else {
4455            return Some(Principal::Direct);
4456        };
4457        self.supervisor
4458            .spawned_consumer_authorized(
4459                &consumer_identity.module_id,
4460                &consumer_identity.launch_nonce,
4461            )
4462            .then(|| Principal::Reserved {
4463                module_id: consumer_identity.module_id.clone(),
4464            })
4465    }
4466
4467    /// The caller fields for a refusal that only has the raw request frame,
4468    /// such as the connection-level admission refusal sent before the request
4469    /// reaches [`Self::handle_route_open`].
4470    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4471        let Ok(ClientControlRequest::RouteOpen {
4472            identity,
4473            consumer_identity,
4474            scope,
4475            ..
4476        }) = parse_client_control_request(&frame.body)
4477        else {
4478            return RouteOpenCaller::unknown();
4479        };
4480        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4481        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4482    }
4483
4484    fn route_open_principal(
4485        &self,
4486        frame: &Frame,
4487        consumer_identity: Option<ConsumerIdentity>,
4488    ) -> Result<Result<Principal, Frame>, RouterError> {
4489        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4490        self.route_open_principal_attested(frame, consumer_identity, attested)
4491    }
4492
4493    /// [`Self::route_open_principal`] with `attested` already taken by
4494    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4495    /// open checks the launch nonce once.
4496    fn route_open_principal_attested(
4497        &self,
4498        frame: &Frame,
4499        consumer_identity: Option<ConsumerIdentity>,
4500        attested: Option<Principal>,
4501    ) -> Result<Result<Principal, Frame>, RouterError> {
4502        if let Some(principal) = attested {
4503            return Ok(Ok(principal));
4504        }
4505        let Some(consumer_identity) = consumer_identity else {
4506            return Ok(Ok(Principal::Direct));
4507        };
4508
4509        Ok(Err(control_error_frame(
4510            frame,
4511            "bad_consumer_identity",
4512            format!(
4513                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4514                consumer_identity.module_id
4515            ),
4516        )?))
4517    }
4518
4519    /// Ordinary `route.open` refusals go through here; admission and breaker
4520    /// refusals log separately with their capacity or breaker state. The daemon can
4521    /// attest which code it sent: without the event, a client's "the daemon
4522    /// refused me" and the daemon's own view could only be reconciled by
4523    /// argument. Malformed input (`invalid_project_root`) does not come here;
4524    /// rejecting a request that was never a valid open is not a refusal of one.
4525    #[allow(clippy::too_many_arguments)]
4526    fn route_open_refusal_frame(
4527        &self,
4528        ctx: &RouteCtx,
4529        caller: &RouteOpenCaller,
4530        frame: &Frame,
4531        module_id: &str,
4532        reason: &'static str,
4533        code: &'static str,
4534        message: impl Into<String>,
4535    ) -> Result<Frame, RouterError> {
4536        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4537        control_error_frame(frame, code, message.into())
4538    }
4539
4540    /// Answer a waiting `route.open` whose relayed bind was refused.
4541    ///
4542    /// `daemon_authored` is provenance, not a code match: the daemon
4543    /// synthesized the refusal itself (a drain, a late reply, a scope that
4544    /// ended mid-bind) or the module refused the bind with its own body. A
4545    /// daemon refusal carries the daemon-origin bit; a module's own
4546    /// rejection is relayed with the bit clear, so a module that refuses
4547    /// with `module_reloading` still cannot mint a daemon refusal.
4548    #[allow(clippy::too_many_arguments)]
4549    fn route_bind_rejection(
4550        &self,
4551        ctx: &RouteCtx,
4552        caller: &RouteOpenCaller,
4553        frame: &Frame,
4554        target_module_id: &str,
4555        body: ErrorBody,
4556        daemon_authored: bool,
4557    ) -> Result<Vec<Frame>, RouterError> {
4558        // The daemon's own commit re-check refused the bind because the
4559        // scope ended or changed after admission. The module accepted;
4560        // counting it as a module rejection would blame the module. This
4561        // branch only runs for daemon-synthesized bodies: a module that
4562        // refuses with a scope code is answered as a module rejection below.
4563        if daemon_authored {
4564            let scope_code = match body.code.as_str() {
4565                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4566                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4567                _ => None,
4568            };
4569            if let Some(code) = scope_code {
4570                self.observe_route_open_refusal(
4571                    ctx,
4572                    caller,
4573                    target_module_id,
4574                    "scope_changed_before_commit",
4575                    code,
4576                );
4577                return Ok(vec![control_error_body_frame(frame, body)?]);
4578            }
4579        }
4580        self.counters
4581            .increment_route_open_refused("module_rejected");
4582        info!(
4583            target: "control",
4584            code = "module_rejected",
4585            module_code = ?body.code,
4586            module_id = ?target_module_id,
4587            connection_id = ctx.connection_id.get(),
4588            principal = %caller.principal,
4589            scope = %caller.scope,
4590            harness = %caller.harness,
4591            "route.open refused"
4592        );
4593        let refusal = if daemon_authored {
4594            control_error_body_frame(frame, body)?
4595        } else {
4596            control_relayed_error_body_frame(frame, body)?
4597        };
4598        Ok(vec![refusal])
4599    }
4600
4601    /// Refuse a `route.open` because the target module's bind-relay breaker is
4602    /// open, without attempting the relay.
4603    ///
4604    /// The wire code is `module_timeout`, which is the truth (the module has
4605    /// not been answering binds) and which both SDKs already classify as
4606    /// retryable with capped backoff. Reusing it is what keeps this change out
4607    /// of both SDKs; the daemon-side distinction lives in the counter key
4608    /// instead.
4609    ///
4610    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4611    /// While a breaker is open this fires on every open to that module, and the
4612    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4613    /// produced 261 lines about a single module inside 3000 lines of daemon
4614    /// log. The rare transitions are logged at warn/info instead and the volume
4615    /// is carried by the counter, so the evidence survives without the flood.
4616    /// The debug line keeps a per-refusal record reachable for whoever turns
4617    /// the level up.
4618    #[allow(clippy::too_many_arguments)]
4619    fn route_open_breaker_refusal_frame(
4620        &self,
4621        ctx: &RouteCtx,
4622        caller: &RouteOpenCaller,
4623        frame: &Frame,
4624        module_id: &str,
4625        consecutive_timeouts: u32,
4626        retry_in: Duration,
4627        probe_in_flight: bool,
4628    ) -> Result<Frame, RouterError> {
4629        self.counters
4630            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4631        debug!(
4632            target: "control",
4633            code = "module_timeout",
4634            module_id = ?module_id,
4635            connection_id = ctx.connection_id.get(),
4636            principal = %caller.principal,
4637            scope = %caller.scope,
4638            harness = %caller.harness,
4639            consecutive_timeouts,
4640            retry_in_ms = retry_in.as_millis() as u64,
4641            probe_in_flight,
4642            "route.open refused by open bind-relay breaker"
4643        );
4644        // Say what a caller can act on. An open bind-relay breaker means the
4645        // module timed out accepting several new routes in a row. The module
4646        // is still running and its established routes keep working; only new
4647        // route.open requests are refused until the cooldown ends and one
4648        // test route (the probe) gets through. A message that only counts
4649        // failed relays reads as "the module is down" to a worker that sees it.
4650        let detail = if probe_in_flight {
4651            "one test route is already being tried; retry once it settles".to_string()
4652        } else {
4653            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4654        };
4655        control_error_frame(
4656            frame,
4657            "module_timeout",
4658            format!(
4659                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4660                 timed out in a row); {detail}; its established routes are unaffected"
4661            ),
4662        )
4663    }
4664
4665    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4666    /// requester's bytes (an unknown target is whatever the client sent) and
4667    /// is Debug-formatted so control characters land in the log escaped
4668    /// rather than as terminal sequences for whoever tails it.
4669    ///
4670    /// `reason` names the check that refused, because one wire code has
4671    /// several senders: after a module registers, `target_unavailable` can
4672    /// come from a missing role, an inactive registration, a supervisor that
4673    /// has not marked the process live, a missing forwarding connection, or a
4674    /// failed relay, and a log that records only the code cannot say which of
4675    /// them fired. It is a static, daemon-chosen label per branch, so it is
4676    /// safe to print plainly and stays a closed set.
4677    fn observe_route_open_refusal(
4678        &self,
4679        ctx: &RouteCtx,
4680        caller: &RouteOpenCaller,
4681        module_id: &str,
4682        reason: &'static str,
4683        code: &'static str,
4684    ) {
4685        self.counters.increment_route_open_refused(code);
4686        info!(
4687            target: "control",
4688            code,
4689            reason,
4690            module_id = ?module_id,
4691            connection_id = ctx.connection_id.get(),
4692            principal = %caller.principal,
4693            scope = %caller.scope,
4694            harness = %caller.harness,
4695            "route.open refused"
4696        );
4697        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4698            self.route_outages.record_not_serving(module_id, reason);
4699        }
4700    }
4701
4702    /// Record an ACCEPTED route.open.
4703    ///
4704    /// Refusals have been logged and counted since the attestation work; accepts
4705    /// were invisible, so the daemon knew every principal it stamped and wrote
4706    /// none of them down. The party that attests the identity was the only party
4707    /// not recording it, which left a credential vault unable to name the sender
4708    /// of a call that reached it (claustrum #43) and left the launch-nonce
4709    /// concurrency question unanswerable from the outside.
4710    ///
4711    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4712    /// `code`/`module_id`/`connection_id` returns both directions of the same
4713    /// decision rather than two shapes a reader has to join by hand.
4714    ///
4715    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4716    /// and the difference carries information rather than being an
4717    /// inconsistency. This line is only reachable after a successful bind to a
4718    /// REGISTERED module, so the value has already passed HELLO validation
4719    /// including the path-hazard refusal and cannot contain control bytes. A
4720    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4721    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4722    ///
4723    /// Bare is also what every other daemon line already emits (`module
4724    /// registered`, `configured module supervised`). Shipping `?module_id` here
4725    /// made this instrument the only one in the file whose ids did not answer
4726    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4727    /// line whose whole purpose is being grepped beside its sibling.
4728    ///
4729    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4730    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4731    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4732    /// one are recorded identically. A test written against that harness passes
4733    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4734    /// green assertion that cannot fail. The same limit applies to the escaping
4735    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4736    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4737    /// Fencing either needs the real formatter, not the capture layer.
4738    ///
4739    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4740    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4741    /// to record. The ephemeral port would decay within minutes and answer only a
4742    /// live question. The identity question is instead answered by counting
4743    /// distinct live connections presenting one module's `consumer_identity` --
4744    /// "is anyone else holding this secret" rather than "is this the right
4745    /// process".
4746    fn observe_route_open_accept(
4747        &self,
4748        ctx: &RouteCtx,
4749        module_id: &str,
4750        principal: &str,
4751        scope: &str,
4752    ) {
4753        self.route_outages.record_accepted(module_id);
4754        self.counters.increment_route_open_accepted(principal);
4755        info!(
4756            target: "control",
4757            principal,
4758            module_id,
4759            connection_id = ctx.connection_id.get(),
4760            scope = %scope,
4761            "route.open accepted"
4762        );
4763    }
4764
4765    fn supervised_absent_route_open_refusal_frame(
4766        &self,
4767        ctx: &RouteCtx,
4768        caller: &RouteOpenCaller,
4769        frame: &Frame,
4770        module_id: &str,
4771        code: &'static str,
4772        status: &crate::supervise::ModuleStatus,
4773    ) -> Result<Frame, RouterError> {
4774        self.counters.increment_route_open_refused(code);
4775        info!(
4776            target: "control",
4777            code,
4778            reason = "supervised_not_registered",
4779            module_id = ?module_id,
4780            connection_id = ctx.connection_id.get(),
4781            principal = %caller.principal,
4782            scope = %caller.scope,
4783            harness = %caller.harness,
4784            state = %status.state,
4785            enabled = status.enabled,
4786            live = status.live,
4787            "route.open refused"
4788        );
4789        // A supervised module whose process has not registered is not
4790        // serving, whatever the reason; the supervisor knows this id, so it is
4791        // safe to track.
4792        self.route_outages
4793            .record_not_serving(module_id, "supervised_not_registered");
4794        let message = if let Some(pid) = status.waiting_for_previous_pid {
4795            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4796                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4797            })
4798        } else {
4799            format!(
4800                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4801                status.state, status.enabled, status.live
4802            )
4803        };
4804        control_error_frame(frame, code, message)
4805    }
4806
4807    async fn handle_route_open(
4808        &self,
4809        ctx: &RouteCtx,
4810        frame: Frame,
4811        request: RouteOpenRequest,
4812        timings: &mut RouteOpenTimings,
4813    ) -> Result<Vec<Frame>, RouterError> {
4814        let RouteOpenRequest {
4815            target,
4816            mut identity,
4817            consumer_identity,
4818            consumer_capabilities,
4819            role_versions,
4820            busy_refusals,
4821            admission_facts,
4822            scope,
4823        } = request;
4824        let target_module_id = target_module_id(&target).to_string();
4825        timings.module_id.clone_from(&target_module_id);
4826        if self
4827            .registry
4828            .get_module_by_connection(ctx.connection_id)
4829            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4830            .is_some()
4831        {
4832            return Ok(vec![control_error_frame(
4833                &frame,
4834                "invalid_request",
4835                "module connections cannot open client routes",
4836            )?]);
4837        }
4838        debug!(
4839            connection_id = ctx.connection_id.get(),
4840            corr = frame.header.corr,
4841            module_id = %target_module_id,
4842            "handling route.open"
4843        );
4844
4845        // The caller's attested principal, requested scope and harness, for
4846        // every `route.open refused` line below. The launch nonce is checked
4847        // here, once, and the answer is reused where the principal is
4848        // enforced further down, so the order of refusals is unchanged.
4849        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4850        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4851
4852        // A malformed declaration is refused first, before anything about the
4853        // target is looked up: the same body would be refused against any
4854        // module, so the caller learns nothing by retrying or waiting. An empty
4855        // map declares nothing and travels as no field at all, so a provider
4856        // only ever sees a missing field or a non-empty one.
4857        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4858        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4859            self.observe_route_open_refusal(
4860                ctx,
4861                &caller,
4862                &target_module_id,
4863                "invalid_role_versions",
4864                error_codes::INVALID_REQUEST,
4865            );
4866            return Ok(vec![control_error_body_frame(
4867                &frame,
4868                ErrorBody {
4869                    code: error_codes::INVALID_REQUEST.to_string(),
4870                    message: error.to_string(),
4871                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4872                },
4873            )?]);
4874        }
4875
4876        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4877            if status.waiting_for_previous_pid.is_some() {
4878                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4879                    ctx,
4880                    &caller,
4881                    &frame,
4882                    &target_module_id,
4883                    "module_warming",
4884                    &status,
4885                )?]);
4886            }
4887        }
4888
4889        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4890        // opposite. Below, a caller learns whether a module is unregistered,
4891        // supervised-but-down (with state/enabled/live), or registered without the
4892        // requested role. Elsewhere that is an enumeration leak: a probe learning
4893        // the shape of a fleet it cannot otherwise see.
4894        //
4895        // It is not one here, and the reason is the ACCESS MODEL rather than
4896        // anything about these errors. Reaching route.open requires the
4897        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4898        // connection file, so any caller who completes it already runs as this
4899        // user -- and can read subc.jsonc for the module list and `ck module
4900        // status` for live state. The reply discloses nothing the caller cannot
4901        // read more easily from disk, while the precision is load-bearing:
4902        // `unknown_module` is retryable and a missing role is not.
4903        //
4904        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4905        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4906        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4907        let Some(registration) = self
4908            .registry
4909            .get_module(&target_module_id)
4910            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4911        else {
4912            if let Some((status, warming)) =
4913                self.supervisor_status(&target_module_id, frame.header.corr)?
4914            {
4915                // BEFORE the two availability codes below, because for a module
4916                // that speaks no subc wire both of them are false comfort: they
4917                // say "not right now" and are retried, and this module will
4918                // never register no matter how long the caller waits. The
4919                // absence here is the declaration being honoured, not a module
4920                // that is late.
4921                if status.protocol == ModuleProtocol::None {
4922                    return Ok(vec![self.route_open_refusal_frame(
4923                        ctx,
4924                        &caller,
4925                        &frame,
4926                        &target_module_id,
4927                        "protocol_none",
4928                        error_codes::MODULE_NO_PROTOCOL,
4929                        format!(
4930                            "module_id '{target_module_id}' is declared protocol: none; \
4931                             it speaks no subc wire and serves no routes"
4932                        ),
4933                    )?]);
4934                }
4935                let code = if warming {
4936                    "module_warming"
4937                } else {
4938                    "target_unavailable"
4939                };
4940                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4941                    ctx,
4942                    &caller,
4943                    &frame,
4944                    &target_module_id,
4945                    code,
4946                    &status,
4947                )?]);
4948            }
4949            if let Some(removed_ago_ms) =
4950                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4951            {
4952                return Ok(vec![self.route_open_refusal_frame(
4953                    ctx,
4954                    &caller,
4955                    &frame,
4956                    &target_module_id,
4957                    "removed",
4958                    error_codes::MODULE_REMOVED,
4959                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4960                )?]);
4961            }
4962            return Ok(vec![self.route_open_refusal_frame(
4963                ctx,
4964                &caller,
4965                &frame,
4966                &target_module_id,
4967                "not_registered",
4968                error_codes::UNKNOWN_MODULE,
4969                format!("module_id '{target_module_id}' is not registered"),
4970            )?]);
4971        };
4972
4973        // Best-effort only: registry readiness and forwarding reservation use
4974        // different locks, so a module can flip readiness between this read and
4975        // the relay. Modules must still tolerate an `on_bind` while not ready.
4976        if !registration.ready {
4977            self.counters
4978                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4979            info!(
4980                target: "control",
4981                code = error_codes::MODULE_WARMING,
4982                module_id = ?target_module_id,
4983                connection_id = ctx.connection_id.get(),
4984                principal = %caller.principal,
4985                scope = %caller.scope,
4986                harness = %caller.harness,
4987                reason = "declared_not_ready",
4988                "route.open refused"
4989            );
4990            // The module is registered but says it cannot take work, which is
4991            // an outage from the caller's side even though its process is up.
4992            self.route_outages
4993                .record_not_serving(&target_module_id, "declared_not_ready");
4994            return Ok(vec![control_error_body_frame(
4995                &frame,
4996                ErrorBody {
4997                    code: error_codes::MODULE_WARMING.to_string(),
4998                    message: format!(
4999                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5000                    ),
5001                    detail: Some(serde_json::json!({
5002                        "reason": "declared_not_ready"
5003                    })),
5004                },
5005            )?]);
5006        }
5007
5008        // Effective readiness, second half: a module that declares a capability
5009        // `need: required` is not routable while that capability has no
5010        // registered provider. It is enforced HERE, as a retryable routing
5011        // refusal, and deliberately not as spawn ordering or a boot block. The
5012        // module is still started and registered and can make its own calls;
5013        // spawn ordering is a promise that cannot be kept once a provider
5014        // crashes at runtime, and refusing to boot would stop the whole
5015        // machine, including the tools needed to fix its configuration.
5016        //
5017        // "Provided" is the evaluator's verdict, which counts a provider as
5018        // soon as it has REGISTERED, not once it is ready. Two modules that
5019        // require each other's capabilities are therefore both routable once
5020        // both register; counting readiness instead would deadlock them.
5021        //
5022        // Only new opens are refused. Routes already bound when a provider
5023        // goes away stay bound: nothing here tears them down, and the module
5024        // answers them as it can. Like the readiness read above this is
5025        // best-effort against a provider registering or leaving concurrently.
5026        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
5027            self.counters
5028                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5029            info!(
5030                target: "control",
5031                code = error_codes::MODULE_WARMING,
5032                module_id = ?target_module_id,
5033                connection_id = ctx.connection_id.get(),
5034                principal = %caller.principal,
5035                scope = %caller.scope,
5036                harness = %caller.harness,
5037                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5038                capability = %capability,
5039                "route.open refused"
5040            );
5041            return Ok(vec![control_error_body_frame(
5042                &frame,
5043                ErrorBody {
5044                    code: error_codes::MODULE_WARMING.to_string(),
5045                    message: format!(
5046                        "module_id '{target_module_id}' requires capability '{capability}', \
5047                         which no registered module provides; retry"
5048                    ),
5049                    detail: Some(serde_json::json!({
5050                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5051                        "capability": capability,
5052                    })),
5053                },
5054            )?]);
5055        }
5056
5057        if !target_has_required_role(&target, &registration.manifest.provides) {
5058            return Ok(vec![self.route_open_refusal_frame(
5059                ctx,
5060                &caller,
5061                &frame,
5062                &target_module_id,
5063                "role_not_provided",
5064                "target_unavailable",
5065                format!("module_id '{target_module_id}' does not provide the requested target"),
5066            )?]);
5067        }
5068
5069        if registration.state != ChannelState::Active {
5070            return Ok(vec![self.route_open_refusal_frame(
5071                ctx,
5072                &caller,
5073                &frame,
5074                &target_module_id,
5075                "registration_not_active",
5076                "target_unavailable",
5077                format!("module_id '{target_module_id}' is not active"),
5078            )?]);
5079        }
5080
5081        if self
5082            .forwarding
5083            .module_is_draining(&target_module_id)
5084            .map_err(RouterError::Forwarding)?
5085        {
5086            return Ok(vec![self.route_open_refusal_frame(
5087                ctx,
5088                &caller,
5089                &frame,
5090                &target_module_id,
5091                "reloading",
5092                "module_reloading",
5093                format!("module_id '{target_module_id}' is reloading"),
5094            )?]);
5095        }
5096
5097        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5098            process_liveness.process_live(&target_module_id) == Some(false)
5099        }) {
5100            // A module the supervisor is restarting or reloading can still hold
5101            // a registration: the old process before its connection closes, or
5102            // a new one that registered while the supervisor was draining. The
5103            // forwarding table does not see that as draining, but the consumer
5104            // should still be told to retry soon, exactly as for the drain
5105            // above, rather than that the target is unavailable.
5106            if process_liveness.process_replacing(&target_module_id) {
5107                return Ok(vec![self.route_open_refusal_frame(
5108                    ctx,
5109                    &caller,
5110                    &frame,
5111                    &target_module_id,
5112                    "reloading",
5113                    "module_reloading",
5114                    format!("module_id '{target_module_id}' is reloading"),
5115                )?]);
5116            }
5117            return Ok(vec![self.route_open_refusal_frame(
5118                ctx,
5119                &caller,
5120                &frame,
5121                &target_module_id,
5122                "supervisor_not_live",
5123                "target_unavailable",
5124                format!("module_id '{target_module_id}' is not live"),
5125            )?]);
5126        }
5127
5128        if !self
5129            .forwarding
5130            .has_live_module_connection(&target_module_id)
5131            .map_err(RouterError::Forwarding)?
5132        {
5133            return Ok(vec![self.route_open_refusal_frame(
5134                ctx,
5135                &caller,
5136                &frame,
5137                &target_module_id,
5138                "no_forwarding_connection",
5139                "target_unavailable",
5140                format!("module_id '{target_module_id}' has no live forwarding connection"),
5141            )?]);
5142        }
5143
5144        if let Some(error) =
5145            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5146        {
5147            self.observe_route_open_refusal(
5148                ctx,
5149                &caller,
5150                &target_module_id,
5151                "op_not_allowed",
5152                "op_not_allowed",
5153            );
5154            return Ok(vec![error]);
5155        }
5156
5157        let principal =
5158            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5159                Ok(principal) => principal,
5160                Err(error) => {
5161                    self.observe_route_open_refusal(
5162                        ctx,
5163                        &caller,
5164                        &target_module_id,
5165                        "bad_consumer_identity",
5166                        "bad_consumer_identity",
5167                    );
5168                    return Ok(vec![error]);
5169                }
5170            };
5171        timings.principal = principal_label(&principal);
5172
5173        // Only Callosum can attest a production phone build. Consume the claim
5174        // here so it can be kept on the route without reaching any module.
5175        let build_class = identity.build_class.take().filter(|build_class| {
5176            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5177                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5178            } else {
5179                if self
5180                    .build_class_warned
5181                    .lock()
5182                    .unwrap_or_else(|p| p.into_inner())
5183                    .insert(ctx.connection_id)
5184                {
5185                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5186                        "ignoring build_class from a non-Callosum opener");
5187                }
5188                false
5189            }
5190        });
5191
5192        // This is attested, control-plane policy for supervised module origins.
5193        // Keep it before route reservation and out of the opaque forwarding hot
5194        // path: data frames must never acquire a per-frame capability check.
5195        if let Principal::Reserved {
5196            module_id: opening_module_id,
5197        } = &principal
5198        {
5199            if let Some(opening_registration) = self
5200                .registry
5201                .get_module(opening_module_id)
5202                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5203            {
5204                if let Some(capability) =
5205                    denied_capability(&opening_registration.manifest, &registration.manifest)
5206                {
5207                    warn!(
5208                        opening_module_id,
5209                        target_module_id,
5210                        capability,
5211                        "refusing route.open because an attested capability deny edge matches"
5212                    );
5213                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5214                        &frame,
5215                        &target_module_id,
5216                        "capability_deny_edge",
5217                        "capability_forbidden",
5218                        format!(
5219                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5220                        ),
5221                    )?]);
5222                }
5223            }
5224        }
5225
5226        if admission_facts.is_some() {
5227            let carrier_matches = matches!(
5228                &principal,
5229                Principal::Reserved { module_id }
5230                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5231            );
5232            if !carrier_matches {
5233                return Ok(vec![self.route_open_refusal_frame(
5234                    ctx,
5235                    &caller,
5236                    &frame,
5237                    &target_module_id,
5238                    "admission_facts_carrier_not_permitted",
5239                    "admission_facts_not_permitted",
5240                    "admission facts may only be carried by the configured reserved module",
5241                )?]);
5242            }
5243
5244            let target_allowed = self
5245                .admission_facts_targets
5246                .as_ref()
5247                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5248            if !target_allowed {
5249                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5250                    &frame,
5251                    &target_module_id,
5252                    "admission_facts_target_not_listed",
5253                    "admission_facts_target_not_allowed",
5254                    format!(
5255                        "admission facts are not permitted for target module_id '{target_module_id}'"
5256                    ),
5257                )?]);
5258            }
5259
5260            // Keep the value opaque to subc. The downstream admission validator owns
5261            // schema and semantic checks; this daemon only enforces carrier authority
5262            // and the configured destination allowlist.
5263        }
5264
5265        // Scope admission, on the attested principal above and never on the
5266        // request body. The tag read here travels with the pending bind and is
5267        // compared with the published one at commit, so a sync between here
5268        // and the module's ack refuses the open instead of binding a stamp
5269        // that is no longer true.
5270        let (bound_scope, mut scope_stamp) = match &scope {
5271            None => (None, None),
5272            Some(selector) => {
5273                let owner_configured = match &selector.owner {
5274                    // A reserved owner counts as configured from before its
5275                    // process is spawned (see `SupervisorHandle::is_configured`).
5276                    // So an owner that has not synced its scopes yet is refused
5277                    // as retryable (`scope_not_synced`), not as one that will
5278                    // never sync.
5279                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5280                    _ => false,
5281                };
5282                let mut scopes = self
5283                    .scopes
5284                    .write()
5285                    .unwrap_or_else(|poisoned| poisoned.into_inner());
5286                let admitted = scopes
5287                    .admit(&principal, &target_module_id, selector, owner_configured)
5288                    .and_then(|admission| {
5289                        crate::scopes::check_target_flow_support(
5290                            &admission.stamp,
5291                            &target_module_id,
5292                            registration.manifest.capabilities.as_ref(),
5293                        )?;
5294                        crate::scopes::check_target_agent_run_support(
5295                            &admission.stamp,
5296                            &target_module_id,
5297                            registration.manifest.capabilities.as_ref(),
5298                        )?;
5299                        let bound = scopes.reserve_bind(
5300                            &admission,
5301                            &principal,
5302                            ctx.connection_id,
5303                            selector.hold_presence,
5304                        );
5305                        Ok((admission, bound))
5306                    });
5307                match admitted {
5308                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5309                    Err(refusal) => {
5310                        return Ok(vec![self.route_open_refusal_frame(
5311                            ctx,
5312                            &caller,
5313                            &frame,
5314                            &target_module_id,
5315                            refusal.code,
5316                            refusal.code,
5317                            refusal.message,
5318                        )?]);
5319                    }
5320                }
5321            }
5322        };
5323        // From here on a refusal names the scope by the stamp admission
5324        // issued, exactly as the `route.open accepted` line does.
5325        if let Some(stamp) = &scope_stamp {
5326            caller.scope = route_open_scope_label(stamp);
5327        }
5328
5329        // Allow resolvable missing-tail paths so a vanished directory does not
5330        // prevent reopening an existing session for cleanup. Binding does not
5331        // authorize creation of durable provider state.
5332        //
5333        // Resolve real paths rather than lexical cleanup: canonicalize the
5334        // longest existing ancestor and append the missing tail, retaining the
5335        // path's real existing ancestry even when a leaf is missing.
5336        timings.enter(RouteOpenPhase::ProjectRoot);
5337        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5338            Ok(project) => project,
5339            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5340        };
5341        timings.enter(RouteOpenPhase::Admission);
5342        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5343            Ok(root) => root,
5344            Err(err) => {
5345                return Ok(vec![control_error_frame(
5346                    &frame,
5347                    "invalid_project_root",
5348                    err.to_string(),
5349                )?])
5350            }
5351        };
5352        // Compute compatibility project fields before stripping optional stamp
5353        // members. Preserve the canonical opener root and its project ID,
5354        // including absence. For rootless openers, use stamp workdir or `/`, and
5355        // opener ID, else stamp ID, else no ID. Emit these identity fields only
5356        // to targets without bind-no-project/v1.
5357        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5358
5359        // Last gate before any relay work, and deliberately after the cheap
5360        // registry and availability checks above: those name a more precise
5361        // condition (unknown, removed, reloading) and a caller is better served
5362        // by the precise code than by this one.
5363        //
5364        // Everything below this point costs an egress permit, a reserved handle
5365        // pair and, if the module does not answer, the whole relay budget. The
5366        // reader no longer waits for that budget, so cap each target explicitly;
5367        // serial dispatch used to provide the accidental cap of one relay per
5368        // connection. Admission is a mutex-protected count and never waits.
5369        let _concurrency_guard = match self
5370            .route_bind_concurrency
5371            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5372        {
5373            Ok(guard) => guard,
5374            Err(in_flight) => {
5375                return Ok(vec![self.route_open_target_capacity_refusal(
5376                    ctx,
5377                    &caller,
5378                    &frame,
5379                    &target_module_id,
5380                    in_flight,
5381                )?]);
5382            }
5383        };
5384
5385        // A module that has already burned the whole budget `threshold` times
5386        // in a row does not get to charge it again until a probe says it recovered.
5387        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5388            RouteBindAdmission::Admitted { guard, probe } => {
5389                if probe {
5390                    info!(
5391                        module_id = %target_module_id,
5392                        connection_id = ctx.connection_id.get(),
5393                        "route.bind breaker half-open: admitting one probe"
5394                    );
5395                }
5396                guard
5397            }
5398            RouteBindAdmission::Refused {
5399                consecutive_timeouts,
5400                retry_in,
5401                probe_in_flight,
5402            } => {
5403                return Ok(vec![self.route_open_breaker_refusal_frame(
5404                    ctx,
5405                    &caller,
5406                    &frame,
5407                    &target_module_id,
5408                    consecutive_timeouts,
5409                    retry_in,
5410                    probe_in_flight,
5411                )?]);
5412            }
5413        };
5414
5415        // Resolve the per-module budget here so the wait matches the operator's
5416        // intent for this specific target. A per-module override in
5417        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5418        // daemons) wins over the daemon-wide default.
5419        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5420        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5421        let operator_app = self
5422            .operator_apps
5423            .lock()
5424            .unwrap_or_else(|p| p.into_inner())
5425            .get(&ctx.connection_id)
5426            .cloned();
5427        timings.enter(RouteOpenPhase::EgressReservation);
5428        let pending = match self
5429            .forwarding
5430            .begin_route_bind_relay_for(
5431                ctx.connection_id,
5432                ctx.egress.clone(),
5433                response_version(&frame),
5434                frame.header.corr,
5435                &target_module_id,
5436                principal.clone(),
5437                bound_scope,
5438                project_root,
5439                identity.harness.clone(),
5440                build_class,
5441                operator_app.clone(),
5442                relay_deadline,
5443                busy_refusals == Some(true),
5444            )
5445            .await
5446        {
5447            Ok(pending) => pending,
5448            Err(err) => {
5449                return Ok(vec![self.route_open_refusal_frame(
5450                    ctx,
5451                    &caller,
5452                    &frame,
5453                    &target_module_id,
5454                    "relay_reservation_failed",
5455                    forwarding_error_code(&err),
5456                    err.to_string(),
5457                )?])
5458            }
5459        };
5460        timings.enter(RouteOpenPhase::Admission);
5461        let crate::forwarding::PendingRouteBindRelay {
5462            endpoint,
5463            module_sink,
5464            negotiated_ver,
5465            client_channel,
5466            client_epoch,
5467            module_channel,
5468            module_epoch,
5469            corr: relay_corr,
5470            receiver,
5471        } = pending;
5472        let mut reservation =
5473            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5474
5475        // Reserving egress can wait while a module reconnects or a swap cuts
5476        // over. Check the relay's captured connection, not the earlier by-id
5477        // lookup: the original module's flow/run capabilities cannot authorize
5478        // its replacement. The captured sink stays bound to that connection.
5479        let relay_registration = self
5480            .registry
5481            .get_module_by_connection(endpoint.connection_id)
5482            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5483        let relay_capabilities = relay_registration
5484            .as_ref()
5485            .and_then(|registration| registration.manifest.capabilities.as_ref());
5486        if let Some(selector) = &scope {
5487            // Egress reservation may wait across a scope update. Recheck the
5488            // proof against the current record before sending the captured bind.
5489            if let Err(refusal) = self
5490                .scopes
5491                .read()
5492                .unwrap_or_else(|poisoned| poisoned.into_inner())
5493                .check_bind_proof(&principal, selector)
5494            {
5495                reservation.release_and_disarm();
5496                return Ok(vec![self.route_open_refusal_frame(
5497                    ctx,
5498                    &caller,
5499                    &frame,
5500                    &target_module_id,
5501                    refusal.code,
5502                    refusal.code,
5503                    refusal.message,
5504                )?]);
5505            }
5506        }
5507        if let Some(stamp) = scope_stamp
5508            .as_ref()
5509            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5510        {
5511            if let Err(refusal) = crate::scopes::check_target_flow_support(
5512                stamp,
5513                &target_module_id,
5514                relay_capabilities,
5515            )
5516            .and_then(|()| {
5517                crate::scopes::check_target_agent_run_support(
5518                    stamp,
5519                    &target_module_id,
5520                    relay_capabilities,
5521                )
5522            }) {
5523                reservation.release_and_disarm();
5524                return Ok(vec![self.route_open_refusal_frame(
5525                    ctx,
5526                    &caller,
5527                    &frame,
5528                    &target_module_id,
5529                    refusal.code,
5530                    refusal.code,
5531                    refusal.message,
5532                )?]);
5533            }
5534        }
5535
5536        // Older readers may reject unknown fields. Select optional fields from
5537        // the captured endpoint's declaration, including on unscoped binds.
5538        if !relay_capabilities.is_some_and(|caps| {
5539            caps.provides
5540                .iter()
5541                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5542        }) {
5543            if let Some(stamp) = scope_stamp.as_mut() {
5544                stamp.workdir = None;
5545                stamp.project_id = None;
5546            }
5547        }
5548        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5549        // attribution. The run gate above has already required the target to
5550        // declare `worker-runs/v1`, which promises to read both
5551        // `on_behalf_of_agent` and `call_mode`.
5552        let worker_run = scope_stamp
5553            .as_ref()
5554            .is_some_and(crate::scopes::is_worker_run)
5555            && relay_capabilities.is_some_and(|caps| {
5556                caps.provides
5557                    .iter()
5558                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5559            });
5560        if !worker_run
5561            && !relay_capabilities.is_some_and(|caps| {
5562                caps.provides
5563                    .iter()
5564                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5565            })
5566        {
5567            if let Some(stamp) = scope_stamp.as_mut() {
5568                stamp.attributes.on_behalf_of_agent = None;
5569            }
5570        }
5571        if !relay_capabilities.is_some_and(|caps| {
5572            caps.provides
5573                .iter()
5574                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5575        }) {
5576            if let Some(stamp) = scope_stamp.as_mut() {
5577                stamp.attributes.flow_version = None;
5578                stamp.attributes.flow_code_hash = None;
5579                stamp.attributes.flow_install_id = None;
5580            }
5581        }
5582        let call_mode = if worker_run
5583            || relay_capabilities
5584                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5585        {
5586            Some(derive_call_mode(scope_stamp.as_ref()))
5587        } else {
5588            if let Some(stamp) = scope_stamp.as_mut() {
5589                stamp.expires_at_ms = None;
5590            }
5591            None
5592        };
5593
5594        debug!(
5595            connection_id = ctx.connection_id.get(),
5596            client_channel,
5597            client_epoch,
5598            module_channel,
5599            module_epoch,
5600            "reserved route handle pair"
5601        );
5602        // Rendered BEFORE the move into the relay, because the accept arm below
5603        // is where it is logged and the principal is gone by then.
5604        let route_principal_label = principal_label(&principal);
5605        let relay = ModuleControlRequest::RouteBind {
5606            route_channel: module_channel,
5607            epoch: module_epoch,
5608            target,
5609            identity,
5610            principal: Some(principal),
5611            consumer_capabilities,
5612            role_versions,
5613            admission_facts,
5614            scope: scope_stamp,
5615            call_mode,
5616        };
5617        let peer = RouteBindPeer::new()
5618            .with_observed_peer(
5619                relay_capabilities
5620                    .is_some_and(|caps| {
5621                        caps.provides
5622                            .iter()
5623                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5624                    })
5625                    .then(|| {
5626                        self.observed_peers
5627                            .lock()
5628                            .unwrap_or_else(|p| p.into_inner())
5629                            .get(&ctx.connection_id)
5630                            .cloned()
5631                            .unwrap_or_else(ObservedPeer::unobserved)
5632                    }),
5633            )
5634            .with_operator_app(
5635                self.connection_operator_app(operator_app.as_ref())
5636                    .filter(|_| {
5637                        relay_capabilities.is_some_and(|caps| {
5638                            caps.provides
5639                                .iter()
5640                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5641                        })
5642                    }),
5643            );
5644        let relay_body = crate::bind_compat::encode_bind(
5645            &relay,
5646            &peer,
5647            &legacy_bind_project,
5648            relay_capabilities,
5649        )
5650        .map_err(|err| {
5651            RouterError::backend(
5652                0,
5653                frame.header.corr,
5654                format!("failed to encode route.bind request: {err}"),
5655            )
5656        })?;
5657        let relay_frame = Frame::build_with_version(
5658            negotiated_ver,
5659            FrameType::Request,
5660            control_flags(),
5661            0,
5662            0,
5663            relay_corr,
5664            relay_body,
5665        )
5666        .map_err(RouterError::FrameBuild)?;
5667
5668        timings.enter(RouteOpenPhase::BindSend);
5669        let sent = module_sink.send(relay_frame).await;
5670        timings.enter(RouteOpenPhase::Admission);
5671        if let Err(err) = sent {
5672            reservation.release_and_disarm();
5673            return Ok(vec![self.route_open_refusal_frame(
5674                ctx,
5675                &caller,
5676                &frame,
5677                &target_module_id,
5678                "relay_send_failed",
5679                "target_unavailable",
5680                err.to_string(),
5681            )?]);
5682        }
5683
5684        #[cfg(test)]
5685        if let Some(hold) = &self.route_bind_relay_sent_hold {
5686            // First wait: tell the test the bind is queued. Second wait: stay
5687            // here until the test has driven whatever it wants to happen
5688            // between queueing the bind and marking it enqueued.
5689            hold.wait().await;
5690            hold.wait().await;
5691        }
5692
5693        // A missing relay is not by itself an abandoned bind: the module may
5694        // already have acked it and the route may be committed. Only a relay
5695        // dropped without any module reply needs this GOODBYE; one a reply
5696        // settled has already had its GOODBYE decided by the reply path.
5697        if self
5698            .forwarding
5699            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5700            .map_err(RouterError::Forwarding)?
5701            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5702        {
5703            self.send_abandoned_route_bind_goodbye(
5704                &module_sink,
5705                negotiated_ver,
5706                module_channel,
5707                module_epoch,
5708            );
5709        }
5710
5711        timings.enter(RouteOpenPhase::BindAckWait);
5712        let outcome = timeout_at(relay_deadline, receiver).await;
5713        timings.enter(RouteOpenPhase::Admission);
5714        match outcome {
5715            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5716                reservation.disarm();
5717                if breaker.record_accepted() {
5718                    info!(
5719                        module_id = %target_module_id,
5720                        "route.bind breaker closed: the probe was accepted"
5721                    );
5722                }
5723                self.observe_route_open_accept(
5724                    ctx,
5725                    &target_module_id,
5726                    &route_principal_label,
5727                    &caller.scope,
5728                );
5729                Ok(Vec::new())
5730            }
5731            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5732                reservation.release_and_disarm();
5733                // A module that says no in microseconds is healthy. Rejection
5734                // is a different condition with its own refusal and must not
5735                // move the breaker.
5736                breaker.record_inconclusive();
5737                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5738            }
5739            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5740                reservation.release_and_disarm();
5741                breaker.record_inconclusive();
5742                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5743            }
5744            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5745                reservation.release_and_disarm();
5746                breaker.record_inconclusive();
5747                // Fires when the module's connection closes while a relayed
5748                // bind is pending -- typically a caller racing a module restart
5749                // whose bind was relayed BEFORE the drain mark went up. Logged
5750                // because the caller sees only its own error and the fleet has
5751                // already spent one diagnosis round unable to tell this arm
5752                // from a relay timeout without daemon-side evidence.
5753                tracing::warn!(
5754                    module_id = %target_module_id,
5755                    "route.bind relay abandoned: {message}"
5756                );
5757                Ok(vec![self.route_open_refusal_frame(
5758                    ctx,
5759                    &caller,
5760                    &frame,
5761                    &target_module_id,
5762                    "relay_abandoned",
5763                    "target_unavailable",
5764                    message,
5765                )?])
5766            }
5767            Ok(Err(_)) => {
5768                reservation.release_and_disarm();
5769                breaker.record_inconclusive();
5770                Ok(vec![self.route_open_refusal_frame(
5771                    ctx,
5772                    &caller,
5773                    &frame,
5774                    &target_module_id,
5775                    "relay_waiter_canceled",
5776                    "target_unavailable",
5777                    "route.bind relay waiter was canceled before the module responded",
5778                )?])
5779            }
5780            Err(_) => {
5781                reservation.release_and_disarm();
5782                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5783                // answer at all is the one condition a fast refusal can
5784                // usefully stand in for; every other arm already answered.
5785                if let Some(opened) = breaker.record_timeout(
5786                    self.route_bind_breaker_threshold,
5787                    self.route_bind_breaker_cooldown,
5788                ) {
5789                    warn!(
5790                        module_id = %target_module_id,
5791                        consecutive_timeouts = opened.consecutive_timeouts,
5792                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5793                        reopened_after_probe = opened.reopened_after_probe,
5794                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5795                    );
5796                }
5797                // The generous budget just burned to no answer: the module is
5798                // registered and its connection is up, but its bind handler sat
5799                // on the ack for the full budget (warm-on-bind, cold configure,
5800                // or a wedged handler). Every earlier unavailability shape
5801                // fast-refuses BEFORE the relay, so this arm firing means the
5802                // slowness is module-side -- log it so the per-module timeline
5803                // is reconstructable without client audit rows.
5804                tracing::warn!(
5805                    module_id = %target_module_id,
5806                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5807                    "route.bind relay timed out: module did not ack within budget"
5808                );
5809                Ok(vec![self.route_open_refusal_frame(
5810                    ctx,
5811                    &caller,
5812                    &frame,
5813                    &target_module_id,
5814                    "relay_timed_out",
5815                    "module_timeout",
5816                    format!(
5817                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5818                        route_bind_relay_timeout
5819                    ),
5820                )?])
5821            }
5822        }
5823    }
5824
5825    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5826        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5827            snapshot: self.supervisor.spawn_snapshot(),
5828        };
5829        Ok(vec![control_response_body_frame(
5830            &frame,
5831            &response,
5832            "ClientControlResponse::SupervisorSpawnSnapshot",
5833        )?])
5834    }
5835
5836    fn handle_supervisor_spawn_subscribe(
5837        &self,
5838        ctx: &RouteCtx,
5839        frame: Frame,
5840        since: Option<SpawnCursor>,
5841    ) -> Result<Vec<Frame>, RouterError> {
5842        match self.supervisor.subscribe_spawns(
5843            ctx.connection_id,
5844            frame.header.corr,
5845            response_version(&frame),
5846            since,
5847            ctx.egress.clone(),
5848        ) {
5849            Ok(()) => Ok(Vec::new()),
5850            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5851                Ok(vec![control_error_body_frame(
5852                    &frame,
5853                    ErrorBody {
5854                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5855                        message: "spawn cursor belongs to a different daemon incarnation"
5856                            .to_string(),
5857                        detail: Some(serde_json::json!({
5858                            "current_daemon_incarnation": current
5859                        })),
5860                    },
5861                )?])
5862            }
5863            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5864                &frame,
5865                ErrorBody {
5866                    code: "spawn_cursor_too_old".to_string(),
5867                    message: "spawn cursor predates the retained event ring".to_string(),
5868                    detail: Some(serde_json::json!({
5869                        "oldest_retained_cursor": oldest
5870                    })),
5871                },
5872            )?]),
5873            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5874                0,
5875                frame.header.corr,
5876                format!("failed to open supervisor spawn subscription: {error}"),
5877            )),
5878        }
5879    }
5880
5881    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882        let generation = self
5883            .registry
5884            .generation()
5885            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5886        let supervised = self.supervisor.list();
5887        let targets = supervised
5888            .iter()
5889            .map(|module| module.subtree_target())
5890            .collect();
5891        let subtree = tokio::spawn(crate::subtree_resources::read(targets));
5892        let mut modules = Vec::new();
5893        for module in supervised {
5894            let status = module.status_for_control("list").map_err(|err| {
5895                RouterError::backend(
5896                    0,
5897                    frame.header.corr,
5898                    format!("failed to read supervisor status: {err}"),
5899                )
5900            })?;
5901            let (configured, _) = module.configuration().map_err(|err| {
5902                RouterError::backend(
5903                    0,
5904                    frame.header.corr,
5905                    format!("failed to read module configuration: {err}"),
5906                )
5907            })?;
5908            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5909                RouterError::backend(
5910                    0,
5911                    frame.header.corr,
5912                    format!("failed to read retry time: {err}"),
5913                )
5914            })?;
5915            // Status and configuration snapshots release their locks before the image probe awaits.
5916            let image = module.running_image_agreement().await;
5917            // Read per request so the figure is current when the operator asks;
5918            // the daemon samples nothing in between.
5919            let resources = Some(crate::subtree_resources::read_process(module.clone()).await);
5920            let mut verdict =
5921                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5922            verdict.spec = Some(launch_spec_agreement(
5923                &configured,
5924                status.process_alive,
5925                status.launch_spec_digest.as_ref(),
5926            ));
5927            let pending_reload = Some(verdict);
5928            let configured_enabled = self
5929                .capability_evaluator
5930                .configured_enabled(&status.module_id);
5931            // Keep the retired policy field on the wire for one release so
5932            // existing status consumers still receive the platform policy.
5933            modules.push(
5934                SupervisorEntry::new(
5935                    status.module_id,
5936                    status.state.to_string(),
5937                    status.enabled,
5938                    status.live,
5939                    status.health.status,
5940                )
5941                .with_configured_enabled(configured_enabled)
5942                .with_next_retry_at_ms(next_retry_at_ms)
5943                .with_launch_nonce_env(Some(!cfg!(unix)))
5944                .with_launch_nonce_source(status.launch_nonce_source)
5945                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5946                .with_protocol(status.protocol)
5947                .with_pending_reload(pending_reload)
5948                .with_last_probe_ms(status.health.last_probe_ms)
5949                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5950                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5951                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5952                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5953                .with_restart_count(Some(status.restart_count))
5954                .with_max_restarts(Some(status.max_restarts))
5955                .with_lifetime_restarts(Some(status.lifetime_restarts))
5956                .with_spawn_generation(Some(status.spawn_generation))
5957                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5958                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5959                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5960                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5961                .with_resources(resources),
5962            );
5963        }
5964        let readings = subtree
5965            .await
5966            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5967        for (module, reading) in modules.iter_mut().zip(readings) {
5968            if reading.reason.as_deref() == Some("timeout") {
5969                crate::subtree_resources::log_timeout(&module.module_id, "subtree");
5970            }
5971            module.subtree = Some(reading);
5972        }
5973        let response = ClientControlResponse::SupervisorList {
5974            generation,
5975            modules,
5976        };
5977        Ok(vec![control_response_body_frame(
5978            &frame,
5979            &response,
5980            "ClientControlResponse::SupervisorList",
5981        )?])
5982    }
5983
5984    fn handle_supervisor_stderr_tail(
5985        &self,
5986        frame: Frame,
5987        module_id: String,
5988        max_lines: Option<u32>,
5989        max_bytes: Option<u32>,
5990    ) -> Result<Vec<Frame>, RouterError> {
5991        let Some(module) = self.supervisor.get(&module_id) else {
5992            return Ok(vec![control_error_frame(
5993                &frame,
5994                "unknown_module",
5995                format!("module_id '{module_id}' is not supervised"),
5996            )?]);
5997        };
5998
5999        let snapshot = module.stderr_tail(
6000            max_lines.map(|value| value as usize),
6001            max_bytes.map(|value| value as usize),
6002        );
6003
6004        let response = ClientControlResponse::SupervisorStderrTail {
6005            module_id,
6006            tail: StderrTail {
6007                capture: match snapshot.capture {
6008                    CaptureState::Captured => StderrCaptureState::Captured,
6009                    CaptureState::Incomplete { reason } => {
6010                        StderrCaptureState::Incomplete { reason }
6011                    }
6012                    CaptureState::NotCaptured { reason } => {
6013                        StderrCaptureState::NotCaptured { reason }
6014                    }
6015                },
6016                entries: snapshot
6017                    .entries
6018                    .into_iter()
6019                    .map(|entry| match entry {
6020                        TailEntry::Line {
6021                            text,
6022                            truncated,
6023                            at_ms,
6024                        } => StderrTailEntry::Line {
6025                            text,
6026                            truncated,
6027                            at_ms,
6028                        },
6029                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6030                    })
6031                    .collect(),
6032                dropped_lines: snapshot.dropped_lines,
6033            },
6034        };
6035        Ok(vec![control_response_body_frame(
6036            &frame,
6037            &response,
6038            "ClientControlResponse::SupervisorStderrTail",
6039        )?])
6040    }
6041
6042    async fn observe_control_wait<F: std::future::Future>(
6043        &self,
6044        ctx: &RouteCtx,
6045        target: Option<&str>,
6046        kind: &'static str,
6047        future: F,
6048    ) -> F::Output {
6049        self.counters
6050            .observe_reader_wait(
6051                || {
6052                    let module = self
6053                        .forwarding
6054                        .module_id_for_connection(ctx.connection_id)
6055                        .ok()
6056                        .flatten();
6057                    let principal = if module.is_some() { "module" } else { "direct" };
6058                    (
6059                        self.resolve_reader_wait_target(kind, target)
6060                            .or(module)
6061                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6062                        principal.into(),
6063                    )
6064                },
6065                kind,
6066                future,
6067            )
6068            .await
6069    }
6070
6071    /// Resolve the reader-wait module ID only from supervisor or registry
6072    /// records; never attribute a wait from the request selector alone.
6073    pub(crate) fn resolve_reader_wait_target(
6074        &self,
6075        op: &str,
6076        selector: Option<&str>,
6077    ) -> Option<String> {
6078        let selector = selector?;
6079        match op {
6080            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6081                .supervisor
6082                .get(selector)
6083                .map(|module| module.module_id().to_owned()),
6084            ops::SUPERVISOR_HEALTH_PROBE => self
6085                .registry
6086                .get_module(selector)
6087                .ok()
6088                .flatten()
6089                .map(|registration| registration.manifest.module_id),
6090            _ => None,
6091        }
6092    }
6093
6094    async fn handle_supervisor_terminals(
6095        &self,
6096        ctx: &RouteCtx,
6097        frame: Frame,
6098        module_id: String,
6099    ) -> Result<Vec<Frame>, RouterError> {
6100        let Some(module) = self.supervisor.get(&module_id) else {
6101            return Ok(vec![control_error_frame(
6102                &frame,
6103                "unknown_module",
6104                format!("module_id '{module_id}' is not supervised"),
6105            )?]);
6106        };
6107
6108        // The journal read runs on a blocking thread: it can be megabytes of
6109        // file I/O and must not occupy a runtime worker.
6110        let terminals = self
6111            .observe_control_wait(
6112                ctx,
6113                Some(module.module_id()),
6114                ops::SUPERVISOR_TERMINALS,
6115                module.read_durable_terminal_history(),
6116            )
6117            .await
6118            .map_err(|error| {
6119                RouterError::backend(
6120                    0,
6121                    frame.header.corr,
6122                    format!("failed to read terminal history: {error}"),
6123                )
6124            })?;
6125        let response = ClientControlResponse::SupervisorTerminals {
6126            module_id,
6127            terminals,
6128        };
6129        Ok(vec![control_response_body_frame(
6130            &frame,
6131            &response,
6132            "ClientControlResponse::SupervisorTerminals",
6133        )?])
6134    }
6135
6136    fn handle_supervisor_routes(
6137        &self,
6138        frame: Frame,
6139        module_id: Option<String>,
6140    ) -> Result<Vec<Frame>, RouterError> {
6141        let modules = self
6142            .forwarding
6143            .route_census(module_id.as_deref())
6144            .map_err(RouterError::Forwarding)?
6145            .into_iter()
6146            .map(|(module_id, routes)| SupervisorRouteModule {
6147                module_id,
6148                routes: routes
6149                    .into_iter()
6150                    .map(|route| SupervisorRoute {
6151                        consumer: match route.principal {
6152                            Principal::Reserved { module_id } => {
6153                                SupervisorRouteConsumer::Reserved { module_id }
6154                            }
6155                            Principal::Direct | Principal::Unverified => {
6156                                SupervisorRouteConsumer::Direct {
6157                                    connection_id: route.goodbye_target.connection_id.get(),
6158                                }
6159                            }
6160                        },
6161                        age_ms: Instant::now()
6162                            .saturating_duration_since(route.bound_at)
6163                            .as_millis()
6164                            .try_into()
6165                            .unwrap_or(u64::MAX),
6166                        draining: route.draining,
6167                        drain_reason: route.drain_reason,
6168                    })
6169                    .collect(),
6170            })
6171            .collect();
6172        let response = ClientControlResponse::SupervisorRoutes { modules };
6173        Ok(vec![control_response_body_frame(
6174            &frame,
6175            &response,
6176            "ClientControlResponse::SupervisorRoutes",
6177        )?])
6178    }
6179
6180    async fn handle_supervisor_provenance(
6181        &self,
6182        ctx: &RouteCtx,
6183        frame: Frame,
6184        module_id: Option<String>,
6185    ) -> Result<Vec<Frame>, RouterError> {
6186        let targeted = module_id.is_some();
6187        let selected = if let Some(module_id) = module_id {
6188            let Some(module) = self.supervisor.get(&module_id) else {
6189                return Ok(vec![control_error_frame(
6190                    &frame,
6191                    "unknown_module",
6192                    format!("module_id '{module_id}' is not supervised"),
6193                )?]);
6194            };
6195            vec![module]
6196        } else {
6197            self.supervisor.list()
6198        };
6199
6200        let target = targeted.then(|| selected[0].module_id());
6201        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6202            let mut modules = Vec::with_capacity(selected.len());
6203            for module in &selected {
6204                let (status, observed_image) = module
6205                    .status_and_running_image_agreement()
6206                    .await
6207                    .map_err(|err| {
6208                        RouterError::backend(
6209                            0,
6210                            frame.header.corr,
6211                            format!("failed to read supervisor status: {err}"),
6212                        )
6213                    })?;
6214                let module_declared = self
6215                    .registry
6216                    .get_module(&status.module_id)
6217                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6218                    .and_then(|registration| registration.manifest.provenance)
6219                    .map(|build| ModuleDeclaredProvenance::Reported { build })
6220                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6221                #[cfg(test)]
6222                let running_image = match &self.provenance_probe_override {
6223                    Some(result) => result.clone(),
6224                    None => observed_image,
6225                };
6226                #[cfg(not(test))]
6227                let running_image = observed_image;
6228                modules.push(SupervisorModuleProvenance {
6229                    module_id: status.module_id,
6230                    module_declared,
6231                    daemon_observed: SupervisorObservedProcess {
6232                        launch_nonce_source: status.launch_nonce_source,
6233                        pid: status.pid,
6234                        spawned_at_ms: status.spawned_at_ms,
6235                        spawned_from: status.spawned_from,
6236                        running_image,
6237                    },
6238                });
6239            }
6240            let daemon = SupervisorDaemonProvenance {
6241                daemon_build: self.daemon_provenance.build.clone(),
6242                daemon_observed: DaemonObservedProcess {
6243                    pid: self.daemon_provenance.pid,
6244                    started_at_ms: self
6245                        .daemon_provenance
6246                        .start_clock
6247                        .map(|clock| clock.started_at_ms())
6248                        .or(self.daemon_provenance.started_at_ms),
6249                    running_image: self
6250                        .daemon_provenance
6251                        .probe
6252                        .observe(
6253                            self.daemon_provenance.pid,
6254                            self.daemon_provenance.executable_path.as_deref(),
6255                            self.daemon_provenance.executable_identity.clone(),
6256                            self.daemon_provenance.process_start_time,
6257                        )
6258                        .await,
6259                },
6260            };
6261            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6262            Ok(vec![control_response_body_frame(
6263                &frame,
6264                &response,
6265                "ClientControlResponse::SupervisorProvenance",
6266            )?])
6267        })
6268        .await
6269    }
6270
6271    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6272        self.refresh_capability_requirements();
6273        let generation = self
6274            .registry
6275            .generation()
6276            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6277        let modules = self
6278            .supervisor
6279            .list()
6280            .into_iter()
6281            .map(|module| {
6282                let status = module.status_for_control("health").map_err(|err| {
6283                    RouterError::backend(
6284                        0,
6285                        frame.header.corr,
6286                        format!("failed to read supervisor health: {err}"),
6287                    )
6288                })?;
6289                let module_id = status.module_id;
6290                let capability_detail = self
6291                    .capability_evaluator
6292                    .required_problem_detail(&module_id);
6293                Ok(SupervisorHealthEntry {
6294                    module_id,
6295                    status: status.health.status,
6296                    detail: append_capability_problem_detail(
6297                        status.health.detail,
6298                        capability_detail,
6299                    ),
6300                    metrics: status.health.metrics,
6301                    consecutive_failures: status.health.consecutive_failures,
6302                    late_answer_count: status.health.late_answer_count,
6303                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6304                    last_action: status.health.last_action,
6305                    last_action_ms: status.health.last_action_ms,
6306                    last_probe_ms: status.health.last_probe_ms,
6307                })
6308            })
6309            .collect::<Result<Vec<_>, RouterError>>()?;
6310        let response = ClientControlResponse::SupervisorHealth {
6311            generation,
6312            modules,
6313        };
6314        Ok(vec![control_response_body_frame(
6315            &frame,
6316            &response,
6317            "ClientControlResponse::SupervisorHealth",
6318        )?])
6319    }
6320
6321    async fn handle_supervisor_restart(
6322        &self,
6323        frame: Frame,
6324        module_id: String,
6325        drain_timeout_ms: Option<u64>,
6326        requester: Principal,
6327    ) -> Result<Vec<Frame>, RouterError> {
6328        let operation_lock = self.supervisor.operation_lock();
6329        let _operation_guard = operation_lock.lock().await;
6330        let Some(module) = self.supervisor.get(&module_id) else {
6331            return Ok(vec![control_error_frame(
6332                &frame,
6333                "unknown_module",
6334                format!("module_id '{module_id}' is not supervised"),
6335            )?]);
6336        };
6337
6338        self.route_outages.mark_operator_action(&module_id);
6339        if let Err(err) = module
6340            .restart_requested(drain_timeout_ms, Some(requester))
6341            .await
6342        {
6343            self.route_outages
6344                .operator_action_ended_unrefused(&module_id);
6345            let (code, message) = match err {
6346                crate::supervise::SuperviseError::Disabled { .. } => {
6347                    ("module_disabled", err.to_string())
6348                }
6349                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6350                    ("swap_in_progress", err.to_string())
6351                }
6352                _ => (
6353                    "target_unavailable",
6354                    format!("failed to restart module_id '{module_id}': {err}"),
6355                ),
6356            };
6357            return Ok(vec![control_error_frame(&frame, code, message)?]);
6358        }
6359
6360        let response = ClientControlResponse::SupervisorAck {
6361            module_id,
6362            applied: true,
6363        };
6364        Ok(vec![control_response_body_frame(
6365            &frame,
6366            &response,
6367            "ClientControlResponse::SupervisorAck",
6368        )?])
6369    }
6370
6371    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
6372    /// when the old process has finished draining: a caller whose own lane
6373    /// rides the old process must get its reply before that drain waits on it.
6374    async fn handle_supervisor_swap(
6375        &self,
6376        frame: Frame,
6377        module_id: String,
6378        ready_timeout_ms: Option<u64>,
6379        requester: Principal,
6380    ) -> Result<Vec<Frame>, RouterError> {
6381        // The daemon-wide operation lock is held only to resolve the handle,
6382        // not across the swap. The swap can take its whole readiness budget,
6383        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6384        // holding it here would park an operator's stop behind the swap it is
6385        // meant to abort. A rescan or stop that reaches the module during the
6386        // swap is served by the swap itself (see `supervise_swap`).
6387        let module = {
6388            let operation_lock = self.supervisor.operation_lock();
6389            let _operation_guard = operation_lock.lock().await;
6390            self.supervisor.get(&module_id)
6391        };
6392        let Some(module) = module else {
6393            return Ok(vec![control_error_frame(
6394                &frame,
6395                "unknown_module",
6396                format!("module_id '{module_id}' is not supervised"),
6397            )?]);
6398        };
6399
6400        self.route_outages.mark_operator_action(&module_id);
6401        if let Err(err) = module
6402            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6403            .await
6404        {
6405            self.route_outages
6406                .operator_action_ended_unrefused(&module_id);
6407            use crate::supervise::SuperviseError;
6408            let message = err.to_string();
6409            let error = match err {
6410                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6411                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6412                    code: "swap_refused".to_string(),
6413                    message,
6414                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6415                },
6416                SuperviseError::SwapFailed {
6417                    arm,
6418                    candidate_exit,
6419                    ..
6420                } => ErrorBody {
6421                    code: "swap_failed".to_string(),
6422                    message,
6423                    detail: Some(serde_json::json!({
6424                        "arm": arm.as_str(),
6425                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6426                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6427                    })),
6428                },
6429                _ => ErrorBody::new(
6430                    "target_unavailable",
6431                    format!("failed to swap module_id '{module_id}': {message}"),
6432                ),
6433            };
6434            return Ok(vec![control_error_body_frame(&frame, error)?]);
6435        }
6436        // A completed swap kept the incumbent serving until cutover, so it
6437        // usually opened no outage; a mark left behind would make the next,
6438        // unrelated outage read as requested.
6439        self.route_outages
6440            .operator_action_ended_unrefused(&module_id);
6441
6442        let response = ClientControlResponse::SupervisorAck {
6443            module_id,
6444            applied: true,
6445        };
6446        Ok(vec![control_response_body_frame(
6447            &frame,
6448            &response,
6449            "ClientControlResponse::SupervisorAck",
6450        )?])
6451    }
6452
6453    async fn handle_supervisor_reload(
6454        &self,
6455        frame: Frame,
6456        module_id: String,
6457        requester: Principal,
6458    ) -> Result<Vec<Frame>, RouterError> {
6459        let operation_lock = self.supervisor.operation_lock();
6460        let _operation_guard = operation_lock.lock().await;
6461        let Some(module) = self.supervisor.get(&module_id) else {
6462            return Ok(vec![control_error_frame(
6463                &frame,
6464                "unknown_module",
6465                format!("module_id '{module_id}' is not supervised"),
6466            )?]);
6467        };
6468
6469        self.route_outages.mark_operator_action(&module_id);
6470        if let Err(err) = module.reload_requested(Some(requester)).await {
6471            self.route_outages
6472                .operator_action_ended_unrefused(&module_id);
6473            let (code, message) = match err {
6474                crate::supervise::SuperviseError::Disabled { .. } => {
6475                    ("module_disabled", err.to_string())
6476                }
6477                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6478                    ("swap_in_progress", err.to_string())
6479                }
6480                _ => (
6481                    "reload_failed",
6482                    format!("failed to reload module_id '{module_id}': {err}"),
6483                ),
6484            };
6485            return Ok(vec![control_error_frame(&frame, code, message)?]);
6486        }
6487
6488        let response = ClientControlResponse::SupervisorAck {
6489            module_id,
6490            applied: true,
6491        };
6492        Ok(vec![control_response_body_frame(
6493            &frame,
6494            &response,
6495            "ClientControlResponse::SupervisorAck",
6496        )?])
6497    }
6498
6499    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6500    /// the configuration comes from and whether to apply it; every mode then
6501    /// runs the same comparison, so a preview describes the apply it previews.
6502    async fn handle_supervisor_rescan(
6503        &self,
6504        frame: Frame,
6505        mode: RescanMode,
6506        requester: Principal,
6507    ) -> Result<Vec<Frame>, RouterError> {
6508        let preview = mode.previews();
6509        let Some(context) = self.rescan.clone() else {
6510            return Ok(vec![control_error_frame(
6511                &frame,
6512                "rescan_unavailable",
6513                "the daemon was not started with a reloadable config path".to_string(),
6514            )?]);
6515        };
6516
6517        let operation_lock = self.supervisor.operation_lock();
6518        let _operation_guard = operation_lock.lock().await;
6519        // Loaded under the operation lock, so no other rescan can change the
6520        // staged configuration between a checked apply's comparison and the
6521        // apply itself.
6522        let loaded = match self.load_rescan_config(&mode, &context) {
6523            Ok(config) => config,
6524            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6525        };
6526        // `load` reports a missing file as Ok(None), which is correct at boot
6527        // (no config, nothing to supervise) and catastrophic here: rescan treats
6528        // "not in the config" as "remove it", so an absent file would read as an
6529        // empty module list and retire the entire running fleet. An editor
6530        // writing via write-new-then-rename, or a half-finished edit, is enough
6531        // to open that window. Refuse instead: a config that cannot be read
6532        // carries no instruction to remove anything.
6533        let Some((config, activity_config)) = loaded else {
6534            return Ok(vec![control_error_frame(
6535                &frame,
6536                "invalid_daemon_config",
6537                format!(
6538                    "daemon config not found at {}; refusing to rescan (an absent config would \
6539                     retire every supervised module)",
6540                    context.config_path.display()
6541                ),
6542            )?]);
6543        };
6544        let (
6545            configured_port,
6546            storage_config,
6547            admission_facts_carrier_module_id,
6548            admission_facts_targets,
6549            scope_authority_owners,
6550            privileged_operator_prompt,
6551            modules,
6552            reserved_capabilities,
6553        ) = (
6554            config.port,
6555            config.storage,
6556            config.admission_facts_carrier_module_id,
6557            config.admission_facts_targets,
6558            config.scope_authority_owners,
6559            config.privileged_operator_prompt,
6560            config.modules,
6561            config.reserved_capabilities,
6562        );
6563
6564        // Collect the sections rescan cannot apply, so the REPLY carries them.
6565        //
6566        // The warning below has always been correct and has always gone only to
6567        // the journal -- addressed to whoever reads logs, while the person who
6568        // just edited the config is looking at the CLI. Naming each section
6569        // individually rather than setting a flag: "something outside modules
6570        // changed" sends the operator back to diffing their own file, which is
6571        // the work this is meant to save.
6572        let mut restart_required = Vec::new();
6573        for section in RestartRequiredSection::ALL {
6574            let changed = match section {
6575                RestartRequiredSection::Port => configured_port != context.configured_port,
6576                RestartRequiredSection::Storage => storage_config != context.storage_config,
6577                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6578                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6579                }
6580                RestartRequiredSection::AdmissionFactsTargets => {
6581                    admission_facts_targets != context.admission_facts_targets
6582                }
6583                RestartRequiredSection::ScopeAuthorityOwners => {
6584                    scope_authority_owners != context.scope_authority_owners
6585                }
6586                RestartRequiredSection::PrivilegedOperatorPrompt => {
6587                    privileged_operator_prompt != context.privileged_operator_prompt
6588                }
6589            };
6590            if changed {
6591                restart_required.push(section.label().to_string());
6592            }
6593        }
6594        if !restart_required.is_empty() {
6595            warn!(
6596                config_path = %context.config_path.display(),
6597                sections = %restart_required.join(", "),
6598                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6599            );
6600        }
6601
6602        for configured in &modules {
6603            if let Err(err) = validate_spec(&configured.module_spec()) {
6604                return Ok(vec![control_error_frame(
6605                    &frame,
6606                    "invalid_daemon_config",
6607                    format!("supervisor rescan rejected daemon config: {err}"),
6608                )?]);
6609            }
6610        }
6611
6612        let configured_capabilities = modules
6613            .iter()
6614            .map(|module| (module.module_id.clone(), module.enabled))
6615            .collect::<Vec<_>>();
6616        let preview_capability_warnings = if preview {
6617            let (_, registrations) = self.runtime_capability_snapshot()?;
6618            let current_modules = self
6619                .supervisor
6620                .list()
6621                .into_iter()
6622                .map(|module| module.module_id().to_string())
6623                .collect::<BTreeSet<_>>();
6624            let resulting_modules = configured_capabilities.clone();
6625            let removed = current_modules
6626                .into_iter()
6627                .filter(|module_id| {
6628                    !resulting_modules
6629                        .iter()
6630                        .any(|(configured_id, _)| configured_id == module_id)
6631                })
6632                .collect::<Vec<_>>();
6633            // A candidate preview also names providers the candidate disables.
6634            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6635            let disabled = match &mode {
6636                RescanMode::Candidate(_) => resulting_modules
6637                    .iter()
6638                    .filter(|(_, enabled)| !enabled)
6639                    .map(|(module_id, _)| module_id.clone())
6640                    .collect::<Vec<_>>(),
6641                _ => Vec::new(),
6642            };
6643            self.capability_evaluator.preview_warnings(
6644                resulting_modules,
6645                &removed,
6646                &disabled,
6647                &registrations,
6648            )
6649        } else {
6650            Vec::new()
6651        };
6652        let reconciled = match &mode {
6653            RescanMode::Candidate(candidate) => self
6654                .preview_candidate(&context, modules, candidate)
6655                .map(|result| (result, Vec::new())),
6656            _ => {
6657                self.reconcile_supervised_modules(
6658                    &context.supervisor,
6659                    modules,
6660                    preview,
6661                    requester.clone(),
6662                )
6663                .await
6664            }
6665        };
6666        let (result, changed) = match reconciled {
6667            Ok(result) => result,
6668            Err(message) => {
6669                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6670            }
6671        };
6672        if !preview {
6673            if let Some(ledger) = &self.activity {
6674                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6675                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6676            }
6677            if let Err(code) = self.configure_activity(activity_config).await {
6678                warn!(
6679                    code,
6680                    "activity retention could not be applied during rescan"
6681                );
6682            }
6683            self.capability_evaluator
6684                .configure(configured_capabilities, reserved_capabilities);
6685            self.capability_evaluator.wake_deadline_loop();
6686            self.refresh_capability_requirements();
6687        }
6688        let mut result = result;
6689        result.restart_required = restart_required;
6690        result.capability_warnings = preview_capability_warnings;
6691        let response = ClientControlResponse::SupervisorRescan { result };
6692        Ok(vec![control_response_body_frame(
6693            &frame,
6694            &response,
6695            "ClientControlResponse::SupervisorRescan",
6696        )?])
6697    }
6698
6699    async fn handle_supervisor_release_reserved(
6700        &self,
6701        frame: Frame,
6702        module_id: String,
6703    ) -> Result<Vec<Frame>, RouterError> {
6704        let Some(context) = self.rescan.clone() else {
6705            return Ok(vec![control_error_frame(
6706                &frame,
6707                "release_unavailable",
6708                "reserved-id release requires a daemon started with a reloadable config path",
6709            )?]);
6710        };
6711        let operation_lock = self.supervisor.operation_lock();
6712        let _operation_guard = operation_lock.lock().await;
6713        let loaded = match crate::daemon_config::load(&context.config_path) {
6714            Ok(Some(config)) => config,
6715            Ok(None) => {
6716                return Ok(vec![control_error_frame(
6717                    &frame,
6718                    "invalid_daemon_config",
6719                    format!(
6720                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6721                        context.config_path.display()
6722                    ),
6723                )?])
6724            }
6725            Err(err) => {
6726                return Ok(vec![control_error_frame(
6727                    &frame,
6728                    "invalid_daemon_config",
6729                    format!("unable to verify reserved-id release against daemon config: {err}"),
6730                )?])
6731            }
6732        };
6733        if loaded
6734            .modules
6735            .iter()
6736            .any(|configured| configured.module_id == module_id)
6737        {
6738            return Ok(vec![control_error_frame(
6739                &frame,
6740                "reserved_module_configured",
6741                format!(
6742                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6743                ),
6744            )?]);
6745        }
6746        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6747            return Ok(vec![control_error_frame(
6748                &frame,
6749                "reserved_gate_not_retained",
6750                format!(
6751                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6752                ),
6753            )?]);
6754        }
6755
6756        let response = ClientControlResponse::SupervisorAck {
6757            module_id,
6758            applied: true,
6759        };
6760        Ok(vec![control_response_body_frame(
6761            &frame,
6762            &response,
6763            "ClientControlResponse::SupervisorAck",
6764        )?])
6765    }
6766
6767    /// Compare the configured module set with the supervised one, changing
6768    /// nothing. The staged spec each supervised module holds is what the
6769    /// configured entry is compared with.
6770    fn plan_rescan(
6771        &self,
6772        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6773    ) -> Result<RescanPlan, String> {
6774        let mut current = BTreeMap::new();
6775        for module in self.supervisor.list() {
6776            let (spec, health) = module.configuration().map_err(|err| {
6777                format!(
6778                    "failed to read configuration for module_id '{}': {err}",
6779                    module.module_id()
6780                )
6781            })?;
6782            let enabled = module
6783                .status()
6784                .map_err(|err| {
6785                    format!(
6786                        "failed to read status for module_id '{}': {err}",
6787                        module.module_id()
6788                    )
6789                })?
6790                .enabled;
6791            current.insert(
6792                module.module_id().to_string(),
6793                (module, spec, health, enabled),
6794            );
6795        }
6796        let configured = configured_modules
6797            .into_iter()
6798            .map(|module| (module.module_id.clone(), module))
6799            .collect::<BTreeMap<_, _>>();
6800
6801        let added = configured
6802            .keys()
6803            .filter(|module_id| !current.contains_key(*module_id))
6804            .cloned()
6805            .collect::<Vec<_>>();
6806        let removed = current
6807            .keys()
6808            .filter(|module_id| !configured.contains_key(*module_id))
6809            .cloned()
6810            .collect::<Vec<_>>();
6811        let mut changed_pending_reload = Vec::new();
6812        let mut configuration_changes = BTreeSet::new();
6813        let mut enabled_changes = BTreeSet::new();
6814        let mut unchanged = 0_u32;
6815
6816        for (module_id, configured_module) in &configured {
6817            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6818            else {
6819                continue;
6820            };
6821            // Compare the whole launch spec so a future launch field cannot
6822            // accidentally become a live-only policy change. Health is stored
6823            // separately and applies live without replacing the process.
6824            let launch_changed = *current_spec != configured_module.module_spec();
6825            let configuration_changed =
6826                launch_changed || *current_health != configured_module.health;
6827            // Applying a rescan also replaces the recorded config value, even
6828            // when an operator already put the runtime in the requested state.
6829            let enabled_changed = *current_enabled != configured_module.enabled
6830                || self
6831                    .capability_evaluator
6832                    .configured_enabled(module_id)
6833                    .is_some_and(|enabled| enabled != configured_module.enabled);
6834            if configuration_changed {
6835                configuration_changes.insert(module_id.clone());
6836            }
6837            if launch_changed {
6838                changed_pending_reload.push(module_id.clone());
6839            }
6840            if enabled_changed {
6841                enabled_changes.insert(module_id.clone());
6842            }
6843            if !configuration_changed && !enabled_changed {
6844                unchanged = unchanged.saturating_add(1);
6845            }
6846        }
6847        Ok(RescanPlan {
6848            current,
6849            configured,
6850            added,
6851            removed,
6852            changed_pending_reload,
6853            configuration_changes,
6854            enabled_changes,
6855            unchanged,
6856        })
6857    }
6858
6859    /// Reconcile the running module set against the configured one.
6860    ///
6861    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6862    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6863    /// deliberately shares this function with the executing path rather than
6864    /// computing the same diff somewhere else -- two implementations of one
6865    /// decision agree until they do not, and the whole value of a preview is that
6866    /// it describes the operation that will actually run.
6867    async fn reconcile_supervised_modules(
6868        &self,
6869        supervisor: &Supervisor,
6870        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6871        preview: bool,
6872        requester: Principal,
6873    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6874        let plan = self.plan_rescan(configured_modules)?;
6875
6876        // Everything above this point is pure computation over two snapshots.
6877        // Everything below MUTATES. The preview returns here so the boundary is a
6878        // single early return rather than a condition repeated at each mutation
6879        // site, where one missed guard would apply part of a change the caller was
6880        // told would not happen.
6881        if preview {
6882            return Ok((plan.preview_result(), Vec::new()));
6883        }
6884        let changed = plan
6885            .configuration_changes
6886            .union(&plan.enabled_changes)
6887            .cloned()
6888            .collect::<Vec<_>>();
6889        let RescanPlan {
6890            current,
6891            configured,
6892            added,
6893            removed,
6894            changed_pending_reload,
6895            configuration_changes,
6896            enabled_changes,
6897            unchanged,
6898        } = plan;
6899        // The staged generation moves before the first change rather than
6900        // after the last, so a rescan that fails partway still invalidates
6901        // every earlier `supervisor.rescan_checked` preview: part of its diff
6902        // may have run.
6903        if !(added.is_empty()
6904            && removed.is_empty()
6905            && configuration_changes.is_empty()
6906            && enabled_changes.is_empty())
6907        {
6908            if let Some(context) = &self.rescan {
6909                context
6910                    .generation
6911                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6912            }
6913        }
6914
6915        for module_id in &removed {
6916            let module = &current
6917                .get(module_id)
6918                .expect("removed module came from current supervisor state")
6919                .0;
6920            module
6921                .retire_requested(Some(requester.clone()))
6922                .await
6923                .map_err(|err| {
6924                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6925                })?;
6926            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6927            //
6928            // `handle_route_open` resolves an absent module in three steps:
6929            // registry, then supervisor status, then tombstone. Retiring first
6930            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6931            // went with the teardown above, the supervisor entry went with
6932            // `retire`, and the tombstone does not exist yet -- so a route.open
6933            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6934            // it") for a module that was deliberately removed and whose caller
6935            // should get `module_removed` (TERMINAL, carrying a removal age).
6936            //
6937            // Writing the tombstone first closes it: during the window the
6938            // supervisor entry still answers, so the caller gets
6939            // `target_unavailable` -- retryable, and TRUE, because the module
6940            // is mid-teardown. After both statements it is `module_removed`.
6941            // No instant remains where a removed module reads as one that
6942            // never existed.
6943            //
6944            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6945            // the absence of a test beside a fix invites deletion: these are
6946            // two sync statements with no await between them, so reaching the
6947            // window needs a second worker thread to land exactly between them
6948            // and there is no hook to force it. MEASURED: the 25 daemon_config
6949            // tests pass identically with the old order and the new one, so
6950            // the existing suite cannot see this and a green run is not
6951            // evidence either way. What the suite does hold is the
6952            // post-condition -- a removed module answers `module_removed` --
6953            // which this preserves.
6954            //
6955            // Found by an Athena panel reading the shipped tree against a
6956            // design note (2026-09-19), as the one concrete instance of that
6957            // note's class that survived contact with source. Direction is
6958            // benign: retryable where terminal was intended, never the reverse.
6959            self.supervisor.record_rescan_removal(module_id);
6960            self.supervisor.retire(module_id);
6961            self.route_outages.forget(module_id);
6962        }
6963
6964        for module_id in configured.keys() {
6965            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
6966                continue;
6967            };
6968            let configured_module = configured
6969                .get(module_id)
6970                .expect("configured module id came from configured map");
6971            if configuration_changes.contains(module_id) {
6972                module
6973                    .update_configuration(
6974                        configured_module.module_spec(),
6975                        configured_module.health.clone(),
6976                        configured_module.drain_timeout_ms,
6977                    )
6978                    .await
6979                    .map_err(|err| {
6980                        format!(
6981                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6982                        )
6983                    })?;
6984            }
6985            // Config-only drift is applied by the evaluator's config update;
6986            // do not turn it into a runtime start or stop.
6987            if *current_enabled != configured_module.enabled {
6988                // A rescan that starts or stops a module applies an operator's
6989                // edit to the config, so the resulting outage was asked for.
6990                self.route_outages.mark_operator_action(module_id);
6991                module
6992                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6993                    .await
6994                    .map_err(|err| {
6995                        self.route_outages.operator_action_ended_unrefused(module_id);
6996                        format!(
6997                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6998                            configured_module.enabled
6999                        )
7000                    })?;
7001            }
7002        }
7003
7004        for module_id in &added {
7005            let configured_module = configured
7006                .get(module_id)
7007                .expect("added module id came from configured map");
7008            supervisor
7009                .supervise_configured_requested(
7010                    configured_module.module_spec(),
7011                    configured_module.enabled,
7012                    configured_module.health.clone(),
7013                    configured_module.drain_timeout_ms,
7014                    configured_module.restart,
7015                    configured_module.register_timeout,
7016                    Some(requester.clone()),
7017                )
7018                .map_err(|err| {
7019                    format!("failed to add module_id '{module_id}' during rescan: {err}")
7020                })?;
7021        }
7022
7023        Ok((
7024            SupervisorRescanResult {
7025                added,
7026                removed,
7027                changed_pending_reload,
7028                enabled_changes: enabled_changes.iter().cloned().collect(),
7029                unchanged,
7030                preview: false,
7031                // Filled by the caller, which is the only layer that can see the
7032                // previous config to diff against.
7033                restart_required: Vec::new(),
7034                capability_warnings: Vec::new(),
7035                module_actions: Vec::new(),
7036                earlier_pending: Vec::new(),
7037                expect: None,
7038            },
7039            changed,
7040        ))
7041    }
7042
7043    async fn handle_supervisor_set_enabled(
7044        &self,
7045        frame: Frame,
7046        module_id: String,
7047        enabled: bool,
7048        requester: Principal,
7049    ) -> Result<Vec<Frame>, RouterError> {
7050        let operation_lock = self.supervisor.operation_lock();
7051        let _operation_guard = operation_lock.lock().await;
7052        let Some(module) = self.supervisor.get(&module_id) else {
7053            return Ok(vec![control_error_frame(
7054                &frame,
7055                "unknown_module",
7056                format!("module_id '{module_id}' is not supervised"),
7057            )?]);
7058        };
7059
7060        // Enabling counts as well as disabling: a module an operator starts
7061        // is refused until it registers, and that wait was asked for.
7062        self.route_outages.mark_operator_action(&module_id);
7063        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7064            Ok(applied) => applied,
7065            Err(err) => {
7066                self.route_outages
7067                    .operator_action_ended_unrefused(&module_id);
7068                return Ok(vec![control_error_frame(
7069                    &frame,
7070                    "target_unavailable",
7071                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7072                )?]);
7073            }
7074        };
7075        if !applied {
7076            // Already in the requested state: nothing was made unavailable,
7077            // so the mark must not outlive this request.
7078            self.route_outages
7079                .operator_action_ended_unrefused(&module_id);
7080        }
7081
7082        self.capability_evaluator.wake_deadline_loop();
7083        self.refresh_capability_requirements();
7084        let response = ClientControlResponse::SupervisorAck { module_id, applied };
7085        Ok(vec![control_response_body_frame(
7086            &frame,
7087            &response,
7088            "ClientControlResponse::SupervisorAck",
7089        )?])
7090    }
7091
7092    async fn handle_supervisor_health_probe(
7093        &self,
7094        ctx: &RouteCtx,
7095        frame: Frame,
7096        module_id: String,
7097    ) -> Result<Vec<Frame>, RouterError> {
7098        self.refresh_capability_requirements();
7099        let Some(registration) = self
7100            .registry
7101            .get_module(&module_id)
7102            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7103        else {
7104            return Ok(vec![control_error_frame(
7105                &frame,
7106                "unknown_module",
7107                format!("module_id '{module_id}' is not registered"),
7108            )?]);
7109        };
7110
7111        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
7112        // named for it. Making `module_registration_grants_op` return false
7113        // unconditionally reddens five tests, and every one is named for something
7114        // else -- capability relay, probe/bind demultiplexing, supervision-only
7115        // probing. They exercise a successful advertisement check on the way to their
7116        // own subject.
7117        //
7118        // Real protection, fragile in a specific way: narrowing any of those tests to
7119        // focus on its stated subject would silently remove coverage nobody knows
7120        // they are carrying. Recorded here rather than as a sixth test, because the
7121        // useful fact is WHICH tests hold the guard up -- a new test would add
7122        // coverage without telling the next person what the existing ones quietly do.
7123        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7124        {
7125            return Ok(vec![control_error_frame(
7126                &frame,
7127                "health_not_advertised",
7128                format!("module_id '{module_id}' did not advertise health.check"),
7129            )?]);
7130        }
7131
7132        self.observe_control_wait(
7133            ctx,
7134            Some(&registration.manifest.module_id),
7135            ops::SUPERVISOR_HEALTH_PROBE,
7136            async {
7137                let deadline = Instant::now() + self.health_probe_timeout;
7138                let pending = match self.forwarding.begin_module_control_rpc_for(
7139                    &module_id,
7140                    MODULE_CONTROL_OP_HEALTH_CHECK,
7141                    deadline,
7142                ) {
7143                    Ok(pending) => pending,
7144                    Err(err) => {
7145                        return Ok(vec![control_error_frame(
7146                            &frame,
7147                            forwarding_error_code(&err),
7148                            err.to_string(),
7149                        )?])
7150                    }
7151                };
7152
7153                let PendingModuleControlRpc {
7154                    endpoint,
7155                    module_sink,
7156                    negotiated_ver,
7157                    corr: probe_corr,
7158                    receiver,
7159                } = pending;
7160                let mut guard =
7161                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7162                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7163                    .map_err(|err| {
7164                        RouterError::backend(
7165                            0,
7166                            frame.header.corr,
7167                            format!("failed to encode health.check request: {err}"),
7168                        )
7169                    })?;
7170                let probe_frame = Frame::build_with_version(
7171                    negotiated_ver,
7172                    FrameType::Request,
7173                    control_flags(),
7174                    0,
7175                    0,
7176                    probe_corr,
7177                    probe_body,
7178                )
7179                .map_err(RouterError::FrameBuild)?;
7180
7181                // Queue admission and the reply share one deadline, so a module that
7182                // stops reading cannot park the requesting connection's reader forever.
7183                match timeout_at(deadline, module_sink.send(probe_frame)).await {
7184                    Ok(Ok(())) => {}
7185                    Ok(Err(err)) => {
7186                        return Ok(vec![control_error_frame(
7187                            &frame,
7188                            "target_unavailable",
7189                            err.to_string(),
7190                        )?]);
7191                    }
7192                    Err(_) => {
7193                        return Ok(vec![control_error_frame(
7194                            &frame,
7195                            "module_timeout",
7196                            format!(
7197                                "module_id '{module_id}' did not answer health.check within {:?}",
7198                                self.health_probe_timeout
7199                            ),
7200                        )?]);
7201                    }
7202                }
7203
7204                match timeout_at(deadline, receiver).await {
7205                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7206                        guard.disarm();
7207                        let Some(report) = response.health_report() else {
7208                            return Ok(vec![control_error_frame(
7209                                &frame,
7210                                "invalid_control_body",
7211                                "health.check RPC returned a non-health response",
7212                            )?]);
7213                        };
7214                        // Metrics go out whole here. The supervisor's cached snapshot
7215                        // caps this blob (see truncate_health_metrics), and this path
7216                        // exists precisely to answer without that cap -- so applying it
7217                        // here would leave no way to see what the cached view drops.
7218                        let HealthReport {
7219                            status,
7220                            detail,
7221                            metrics,
7222                        } = report;
7223                        let capability_detail = self
7224                            .capability_evaluator
7225                            .required_problem_detail(&module_id);
7226                        let response = ClientControlResponse::SupervisorHealthProbe {
7227                            module_id,
7228                            status,
7229                            detail: append_capability_problem_detail(detail, capability_detail),
7230                            metrics,
7231                        };
7232                        Ok(vec![control_response_body_frame(
7233                            &frame,
7234                            &response,
7235                            "ClientControlResponse::SupervisorHealthProbe",
7236                        )?])
7237                    }
7238                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7239                        guard.disarm();
7240                        // The module refused the health probe with its own body, so
7241                        // the refusal is relayed without the daemon-origin flag
7242                        // (bit 6): a client must not read it as the daemon's own
7243                        // refusal, which it may treat as never forwarded.
7244                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7245                    }
7246                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7247                        guard.disarm();
7248                        Ok(vec![control_error_frame(
7249                            &frame,
7250                            "target_unavailable",
7251                            message,
7252                        )?])
7253                    }
7254                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7255                        guard.disarm();
7256                        Ok(vec![control_error_frame(
7257                            &frame,
7258                            "invalid_control_body",
7259                            message,
7260                        )?])
7261                    }
7262                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7263                        guard.disarm();
7264                        Ok(vec![control_error_frame(
7265                            &frame,
7266                            "invalid_control_body",
7267                            format!("expected module-control op '{expected}', got '{actual}'"),
7268                        )?])
7269                    }
7270                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7271                        guard.disarm();
7272                        Ok(vec![control_error_frame(
7273                            &frame,
7274                            "module_timeout",
7275                            format!(
7276                                "module_id '{module_id}' answered health.check after {:?}",
7277                                self.health_probe_timeout
7278                            ),
7279                        )?])
7280                    }
7281                    Ok(Err(_)) => Ok(vec![control_error_frame(
7282                        &frame,
7283                        "target_unavailable",
7284                        "health.check waiter was canceled before the module responded",
7285                    )?]),
7286                    Err(_) => Ok(vec![control_error_frame(
7287                        &frame,
7288                        "module_timeout",
7289                        format!(
7290                            "module_id '{module_id}' did not answer health.check within {:?}",
7291                            self.health_probe_timeout
7292                        ),
7293                    )?]),
7294                }
7295            },
7296        )
7297        .await
7298    }
7299
7300    fn supervisor_status(
7301        &self,
7302        module_id: &str,
7303        corr: u64,
7304    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7305        self.supervisor
7306            .get(module_id)
7307            .map(|module| {
7308                let warming = module.is_warming_for_control("status").map_err(|err| {
7309                    RouterError::backend(
7310                        0,
7311                        corr,
7312                        format!(
7313                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
7314                        ),
7315                    )
7316                })?;
7317                module.status_for_control("status").map_err(|err| {
7318                    RouterError::backend(
7319                        0,
7320                        corr,
7321                        format!(
7322                            "failed to read supervisor status for module_id '{module_id}': {err}"
7323                        ),
7324                    )
7325                }).map(|status| (status, warming))
7326            })
7327            .transpose()
7328    }
7329
7330    fn guard_module_control_op(
7331        &self,
7332        frame: &Frame,
7333        module_id: &str,
7334        op: &str,
7335    ) -> Result<Option<Frame>, RouterError> {
7336        if self.module_grants_op(module_id, op, frame.header.corr)? {
7337            return Ok(None);
7338        }
7339
7340        Ok(Some(control_error_frame(
7341            frame,
7342            "op_not_allowed",
7343            format!("module_id '{module_id}' did not grant control op '{op}'"),
7344        )?))
7345    }
7346
7347    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7348        let Some(registration) = self
7349            .registry
7350            .get_module(module_id)
7351            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7352        else {
7353            return Ok(false);
7354        };
7355        Ok(module_registration_grants_op(&registration.control_ops, op))
7356    }
7357
7358    fn handle_status_update(
7359        &self,
7360        endpoint: ModuleEndpointId,
7361        frame: Frame,
7362    ) -> Result<Vec<Frame>, RouterError> {
7363        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7364            Ok(update) => update,
7365            Err(err) => {
7366                // Forward-compat: a newer module may push a channel-0 op this subc
7367                // version doesn't know. The control contract says unknown push ops
7368                // are IGNORED, never answered with an error. Only a malformed body
7369                // for an op we DO know is a real error worth surfacing.
7370                if is_known_module_push_op(&frame.body) {
7371                    return Ok(vec![control_error_frame(
7372                        &frame,
7373                        "invalid_control_body",
7374                        format!("malformed module control push body: {err}"),
7375                    )?]);
7376                }
7377                return Ok(Vec::new());
7378            }
7379        };
7380
7381        match update {
7382            ModuleControlPush::RouteStatus {
7383                route_channel,
7384                route_epoch,
7385                status,
7386            } => {
7387                self.forwarding
7388                    .cache_status(endpoint, route_channel, route_epoch, status)
7389                    .map_err(RouterError::Forwarding)?;
7390            }
7391        }
7392        Ok(Vec::new())
7393    }
7394
7395    fn handle_route_poll(
7396        &self,
7397        ctx: &RouteCtx,
7398        frame: Frame,
7399        route_channel: u16,
7400        route_epoch: u32,
7401        kind: PollKind,
7402    ) -> Result<Vec<Frame>, RouterError> {
7403        let snapshot = self
7404            .forwarding
7405            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7406            .map_err(RouterError::Forwarding)?;
7407        let response = match (kind, snapshot) {
7408            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7409                ClientControlResponse::RoutePoll {
7410                    route_channel,
7411                    route_epoch,
7412                    status,
7413                    live: None,
7414                }
7415            }
7416            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7417                route_channel,
7418                route_epoch,
7419                status: None,
7420                live: None,
7421            },
7422            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7423                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7424                // is what makes reporting `true` correct rather than a
7425                // confident guess. `process_live` returns None only when the
7426                // module id has no supervisor snapshot at all -- an
7427                // externally-started module the daemon did not spawn -- and
7428                // for those the supervisor has no opinion to offer, ever. It
7429                // is never None for a supervised module in an unknown state:
7430                // a supervised module always has a snapshot, and the answer
7431                // comes from `state == Running && process_alive`.
7432                //
7433                // The route is Bound, so the module completed a HELLO on a
7434                // live connection; "the process this route points at is
7435                // running" is therefore attested by the binding rather than
7436                // assumed. Reporting `false` for an unsupervised module would
7437                // be the actual lie -- it would tell a client its healthy
7438                // route is dead because the daemon does not manage the
7439                // process.
7440                //
7441                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7442                // module whose liveness is genuinely unknown, e.g. a snapshot
7443                // that has not been populated yet -- THIS DEFAULT BECOMES
7444                // WRONG and must split: unsupervised stays true, unknown
7445                // becomes null so the client can tell the two apart. The
7446                // response field is already `Option<bool>`, so the wire can
7447                // carry that distinction today.
7448                let live = self
7449                    .process_liveness
7450                    .as_ref()
7451                    .and_then(|source| source.process_live(&module_id))
7452                    .unwrap_or(true);
7453                ClientControlResponse::RoutePoll {
7454                    route_channel,
7455                    route_epoch,
7456                    status: None,
7457                    live: Some(live),
7458                }
7459            }
7460            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7461                route_channel,
7462                route_epoch,
7463                status: None,
7464                live: Some(false),
7465            },
7466        };
7467
7468        Ok(vec![control_response_body_frame(
7469            &frame,
7470            &response,
7471            "ClientControlResponse::RoutePoll",
7472        )?])
7473    }
7474
7475    pub(crate) fn observe_module_control_completion(
7476        &self,
7477        completion: ModuleControlRpcCompletion,
7478    ) -> bool {
7479        match completion {
7480            ModuleControlRpcCompletion::Unknown => false,
7481            ModuleControlRpcCompletion::Settled => true,
7482            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7483                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7484                info!(
7485                    module_id = %module_id,
7486                    latency_ms,
7487                    "late health.check answer proves the module is alive"
7488                );
7489                match self
7490                    .supervisor
7491                    .record_late_health_answer(&module_id, latency_ms)
7492                {
7493                    Ok(true) => {}
7494                    Ok(false) => debug!(
7495                        module_id = %module_id,
7496                        latency_ms,
7497                        "late health.check answer has no active supervisor snapshot"
7498                    ),
7499                    Err(err) => warn!(
7500                        module_id = %module_id,
7501                        latency_ms,
7502                        error = %err,
7503                        "failed to record late health.check answer"
7504                    ),
7505                }
7506                true
7507            }
7508        }
7509    }
7510
7511    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7512    /// the module connection whose frame is being handled, or to the client that
7513    /// relay was opened for.
7514    ///
7515    /// This runs on the MODULE connection's frame handler, where returning `Err`
7516    /// ends that connection -- and a module connection carries every client's
7517    /// routes to that module, so ending it costs the whole fleet its tools.
7518    /// `ConnectionClosing` carries the id of the connection that is closing, and
7519    /// when that id is a CLIENT's, the condition is entirely about that one
7520    /// client's route.open. A client-scoped condition has no authority over a
7521    /// shared module connection, so it is logged and the single relay is dropped:
7522    /// the client is going away, and `complete_pending_relay` already removed the
7523    /// relay before failing, so there is nothing left to settle. Anything that
7524    /// relay still reserved is released by that client's own connection teardown,
7525    /// which is already under way -- that is what "closing" means.
7526    ///
7527    /// Every other failure is a statement about THIS connection and stays fatal:
7528    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7529    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7530    /// frames correctly.
7531    fn refuse_to_end_module_connection_for_a_client(
7532        &self,
7533        module_connection_id: ConnectionId,
7534        corr: u64,
7535        err: ForwardingError,
7536    ) -> Result<(), RouterError> {
7537        if let ForwardingError::ConnectionClosing { connection_id } = err {
7538            if connection_id != module_connection_id {
7539                warn!(
7540                    module_connection_id = module_connection_id.get(),
7541                    client_connection_id = connection_id.get(),
7542                    corr,
7543                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7544                );
7545                return Ok(());
7546            }
7547        }
7548        Err(RouterError::Forwarding(err))
7549    }
7550
7551    fn handle_module_relay_response(
7552        &self,
7553        connection_id: ConnectionId,
7554        frame: Frame,
7555    ) -> Result<Vec<Frame>, RouterError> {
7556        let mut secondary_error = None;
7557        let outcome = match frame.header.ty {
7558            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7559                Ok(probe) if probe.op == "route.bind" => {
7560                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7561                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7562                            RouteBindRelayOutcome::Accepted
7563                        }
7564                        Ok(other) => {
7565                            let message =
7566                                format!("route.bind response carried unexpected body: {other:?}");
7567                            secondary_error = Some(control_error_frame(
7568                                &frame,
7569                                "invalid_control_body",
7570                                message.clone(),
7571                            )?);
7572                            RouteBindRelayOutcome::ModuleGone(message)
7573                        }
7574                        Err(err) => {
7575                            let message = format!("malformed route.bind response body: {err}");
7576                            secondary_error = Some(control_error_frame(
7577                                &frame,
7578                                "invalid_control_body",
7579                                message.clone(),
7580                            )?);
7581                            RouteBindRelayOutcome::ModuleGone(message)
7582                        }
7583                    }
7584                }
7585                Ok(probe) => {
7586                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7587                    {
7588                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7589                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7590                            "malformed {} response body: {err}",
7591                            probe.op
7592                        )),
7593                    };
7594                    let completion = self
7595                        .forwarding
7596                        .complete_module_control_rpc(
7597                            connection_id,
7598                            frame.header.corr,
7599                            Some(&probe.op),
7600                            outcome,
7601                        )
7602                        .map_err(RouterError::Forwarding)?;
7603                    if !self.observe_module_control_completion(completion) {
7604                        debug!(
7605                            connection_id = connection_id.get(),
7606                            corr = frame.header.corr,
7607                            op = %probe.op,
7608                            "dropping late or unknown module-control RPC response"
7609                        );
7610                    }
7611                    return Ok(Vec::new());
7612                }
7613                Err(err) => {
7614                    if let Some(expected_op) = self
7615                        .forwarding
7616                        .pending_module_control_op(connection_id, frame.header.corr)
7617                        .map_err(RouterError::Forwarding)?
7618                    {
7619                        let completion = self
7620                            .forwarding
7621                            .complete_module_control_rpc(
7622                                connection_id,
7623                                frame.header.corr,
7624                                None,
7625                                ModuleControlRpcOutcome::MalformedResponse(format!(
7626                                    "malformed {expected_op} response body: {err}"
7627                                )),
7628                            )
7629                            .map_err(RouterError::Forwarding)?;
7630                        if !self.observe_module_control_completion(completion) {
7631                            debug!(
7632                                connection_id = connection_id.get(),
7633                                corr = frame.header.corr,
7634                                "dropping late malformed module-control RPC response"
7635                            );
7636                        }
7637                        return Ok(Vec::new());
7638                    }
7639                    let message = format!("malformed route.bind response body: {err}");
7640                    secondary_error = Some(control_error_frame(
7641                        &frame,
7642                        "invalid_control_body",
7643                        message.clone(),
7644                    )?);
7645                    RouteBindRelayOutcome::ModuleGone(message)
7646                }
7647            },
7648            FrameType::Error => {
7649                if self
7650                    .forwarding
7651                    .pending_module_control_op(connection_id, frame.header.corr)
7652                    .map_err(RouterError::Forwarding)?
7653                    .is_some()
7654                {
7655                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7656                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7657                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7658                            "malformed module-control ERROR body: {err}"
7659                        )),
7660                    };
7661                    let completion = self
7662                        .forwarding
7663                        .complete_module_control_rpc(
7664                            connection_id,
7665                            frame.header.corr,
7666                            None,
7667                            outcome,
7668                        )
7669                        .map_err(RouterError::Forwarding)?;
7670                    if !self.observe_module_control_completion(completion) {
7671                        debug!(
7672                            connection_id = connection_id.get(),
7673                            corr = frame.header.corr,
7674                            "dropping late or unknown module-control RPC error"
7675                        );
7676                    }
7677                    return Ok(Vec::new());
7678                }
7679                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7680                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7681                    Err(err) => {
7682                        let message = format!("malformed route.bind ERROR body: {err}");
7683                        secondary_error = Some(control_error_frame(
7684                            &frame,
7685                            "invalid_control_body",
7686                            message.clone(),
7687                        )?);
7688                        RouteBindRelayOutcome::ModuleGone(message)
7689                    }
7690                }
7691            }
7692            ty => {
7693                return Ok(vec![control_error_frame(
7694                    &frame,
7695                    "unsupported_control_frame",
7696                    format!("unsupported module channel-0 frame {ty:?}"),
7697                )?])
7698            }
7699        };
7700
7701        let settled =
7702            self.forwarding
7703                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7704        let completion = match settled {
7705            Ok(completion) => completion,
7706            Err(err) => {
7707                self.refuse_to_end_module_connection_for_a_client(
7708                    connection_id,
7709                    frame.header.corr,
7710                    err,
7711                )?;
7712                return Ok(secondary_error.into_iter().collect());
7713            }
7714        };
7715        if let Some(target) = completion.abandoned.as_ref() {
7716            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7717        }
7718        if !completion.settled {
7719            debug!(
7720                connection_id = connection_id.get(),
7721                corr = frame.header.corr,
7722                frame_type = ?frame.header.ty,
7723                "dropping late or unknown route.bind relay response"
7724            );
7725        }
7726        Ok(secondary_error.into_iter().collect())
7727    }
7728
7729    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7730        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7731        // GOODBYE ends the connection's logical session even when its socket
7732        // stays open. Use disconnect teardown so verdicts, client notices and
7733        // scope authority are released at the same lifecycle boundary.
7734        self.cleanup_connection_with_end_reason(
7735            connection_id,
7736            RegistrationEndReason::ExplicitGoodbye,
7737        )
7738        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7739        Ok(Vec::new())
7740    }
7741}
7742
7743impl Default for ControlHandler {
7744    fn default() -> Self {
7745        Self::new(Arc::new(Registry::default()))
7746    }
7747}
7748
7749impl crate::supervise::SwapPromotionObserver for ControlHandler {
7750    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7751        self.apply_registration_capabilities(registration);
7752    }
7753}
7754
7755fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7756    CapabilityRequirementStatus {
7757        consumer: status.consumer,
7758        capability: status.capability,
7759        need: match status.need {
7760            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7761            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7762        },
7763        verdict: status.verdict.as_str().to_string(),
7764        episode_seq: status.episode_seq,
7765        config_satisfiable: status.config_satisfiable,
7766        runtime_available: status.runtime_available,
7767        detail: status.detail,
7768    }
7769}
7770
7771fn append_capability_problem_detail(
7772    detail: Option<String>,
7773    capability_detail: Option<String>,
7774) -> Option<String> {
7775    match (detail, capability_detail) {
7776        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7777        (Some(detail), None) => Some(detail),
7778        (None, Some(capability_detail)) => Some(capability_detail),
7779        (None, None) => None,
7780    }
7781}
7782
7783fn subc_ops() -> Vec<String> {
7784    SUBC_CONTROL_OPS
7785        .iter()
7786        .map(|op| (*op).to_string())
7787        .collect()
7788}
7789
7790fn module_subc_ops() -> Vec<String> {
7791    SUBC_CONTROL_OPS
7792        .iter()
7793        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7794        .map(|op| (*op).to_string())
7795        .collect()
7796}
7797
7798#[cfg(test)]
7799fn module_baseline_control_ops() -> Vec<String> {
7800    MODULE_BASELINE_CONTROL_OPS
7801        .iter()
7802        .map(|op| (*op).to_string())
7803        .collect()
7804}
7805
7806fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7807    let mut seen = HashSet::new();
7808    let mut effective = Vec::new();
7809    for op in MODULE_BASELINE_CONTROL_OPS {
7810        if seen.insert((*op).to_string()) {
7811            effective.push((*op).to_string());
7812        }
7813    }
7814    for op in declared.unwrap_or_default() {
7815        if seen.insert(op.clone()) {
7816            effective.push(op);
7817        }
7818    }
7819    effective
7820}
7821
7822fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7823    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7824}
7825
7826fn target_module_id(target: &RouteTarget) -> &str {
7827    match target {
7828        RouteTarget::ToolProvider { module_id }
7829        | RouteTarget::ManagementSurface { module_id }
7830        | RouteTarget::InternalService { module_id, .. } => module_id,
7831    }
7832}
7833
7834fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7835    roles.iter().any(|role| match (target, role) {
7836        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7837        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7838        (
7839            RouteTarget::InternalService { service_id, .. },
7840            ProviderRole::InternalService {
7841                service_id: provided,
7842                ..
7843            },
7844        ) => service_id == provided,
7845        _ => false,
7846    })
7847}
7848
7849fn is_routable_role(role: &ProviderRole) -> bool {
7850    matches!(
7851        role,
7852        ProviderRole::ToolProvider { .. }
7853            | ProviderRole::ManagementSurface { .. }
7854            | ProviderRole::InternalService { .. }
7855    )
7856}
7857
7858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7859enum ControlRequestBodyError {
7860    UnknownOp,
7861    InvalidBody,
7862}
7863
7864#[derive(Debug, Deserialize)]
7865struct ControlOpProbe {
7866    op: String,
7867}
7868
7869/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7870/// this set is treated as a forward-compat unknown and ignored rather than errored.
7871const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7872
7873fn is_known_module_push_op(body: &[u8]) -> bool {
7874    serde_json::from_slice::<ControlOpProbe>(body)
7875        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7876        .unwrap_or(false)
7877}
7878
7879fn is_known_module_request_op(body: &[u8]) -> bool {
7880    serde_json::from_slice::<ControlOpProbe>(body)
7881        .map(|probe| is_module_to_subc_op(&probe.op))
7882        .unwrap_or(false)
7883}
7884
7885fn is_module_to_subc_op(op: &str) -> bool {
7886    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7887}
7888
7889fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7890    debug!(
7891        op = %op,
7892        connection_id = connection_id.get(),
7893        corr,
7894        "control dispatch"
7895    );
7896}
7897
7898fn log_slow_control_dispatch(
7899    dispatch_started_at: Option<StdInstant>,
7900    op: &'static str,
7901    connection_id: ConnectionId,
7902    corr: u64,
7903    route_open_timings: Option<&RouteOpenTimings>,
7904) {
7905    let Some(dispatch_started_at) = dispatch_started_at else {
7906        return;
7907    };
7908    // Use the same monotonic clock as the phase timers and relay deadline.
7909    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7910    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7911        if let Some(timings) = route_open_timings {
7912            warn!(
7913                op = %op,
7914                connection_id = connection_id.get(),
7915                corr,
7916                elapsed_ms = elapsed.as_millis() as u64,
7917                module_id = ?timings.module_id,
7918                principal = %timings.principal,
7919                admission_ms = timings.admission.as_millis() as u64,
7920                project_root_ms = timings.project_root.as_millis() as u64,
7921                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7922                bind_send_ms = timings.bind_send.as_millis() as u64,
7923                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7924                "slow control dispatch"
7925            );
7926            return;
7927        }
7928        warn!(
7929            op = %op,
7930            connection_id = connection_id.get(),
7931            corr,
7932            elapsed_ms = elapsed.as_millis() as u64,
7933            "slow control dispatch"
7934        );
7935    }
7936}
7937
7938/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7939#[cfg(test)]
7940type TestCallerLookup =
7941    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7942
7943/// Every new client operation needs an explicit privilege decision here.
7944fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7945    match request {
7946        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7947            Some(PrivilegedOperation::Restart { module_id })
7948        }
7949        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7950            Some(PrivilegedOperation::Swap { module_id })
7951        }
7952        ClientControlRequest::SupervisorReload { module_id } => {
7953            Some(PrivilegedOperation::Reload { module_id })
7954        }
7955        ClientControlRequest::SupervisorRescan { preview } => {
7956            if *preview {
7957                None
7958            } else {
7959                Some(PrivilegedOperation::Rescan)
7960            }
7961        }
7962        // Only the well-formed apply asks for approval. A preview changes
7963        // nothing, and any other combination is refused before it touches
7964        // anything, so prompting the operator for it would ask about a request
7965        // that cannot run.
7966        ClientControlRequest::SupervisorRescanChecked {
7967            preview,
7968            candidate,
7969            expect,
7970        } => (!*preview && candidate.is_none() && expect.is_some())
7971            .then_some(PrivilegedOperation::Rescan),
7972        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7973            Some(PrivilegedOperation::SetEnabled {
7974                module_id,
7975                enabled: *enabled,
7976            })
7977        }
7978        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7979            Some(PrivilegedOperation::ReleaseReserved { module_id })
7980        }
7981        ClientControlRequest::ServerDescribe {}
7982        // The announcement uses the same client admission as supervisor.restart,
7983        // but cannot stop anything itself and needs no operator confirmation.
7984        | ClientControlRequest::DaemonShutdownIntent { .. }
7985        | ClientControlRequest::CatalogList { .. }
7986        | ClientControlRequest::RouteOpen { .. }
7987        | ClientControlRequest::RoutePoll { .. }
7988        | ClientControlRequest::SupervisorList {}
7989        | ClientControlRequest::SupervisorHealth {}
7990        | ClientControlRequest::SupervisorHealthProbe { .. }
7991        | ClientControlRequest::SupervisorRoutes { .. }
7992        | ClientControlRequest::SupervisorProvenance { .. }
7993        | ClientControlRequest::SupervisorStderrTail { .. }
7994        | ClientControlRequest::SupervisorTerminals { .. }
7995        | ClientControlRequest::SupervisorSpawnSnapshot {}
7996        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7997    }
7998}
7999
8000fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8001    match request {
8002        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8003        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8004        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8005        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8006        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8007        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8008        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8009        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8010        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8011        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8012        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8013        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8014        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8015        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8016        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8017        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8018        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8019        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8020        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8021        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8022        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8023    }
8024}
8025
8026/// The caller of one `route.open`, as its `route.open refused` line names it.
8027///
8028/// A connection id alone cannot name a caller after the fact: ids restart at
8029/// every daemon restart, so the same number belongs to different modules
8030/// before and after one. These three fields are what the refused line needs
8031/// to be attributable on its own, and they are rendered with the same helpers
8032/// as the `route.open accepted` line so both directions grep alike.
8033struct RouteOpenCaller {
8034    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
8035    /// when the request presented a consumer identity whose launch nonce did
8036    /// not verify. The claimed module id is never printed as a principal.
8037    principal: String,
8038    /// `none` for an unscoped open; the admitted stamp's label once scope
8039    /// admission issued one; before that, the requested selector's label.
8040    scope: String,
8041    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
8042    /// because it is a string the caller chose.
8043    harness: String,
8044}
8045
8046impl RouteOpenCaller {
8047    fn new(
8048        principal: Option<&Principal>,
8049        scope: Option<&ScopeSelector>,
8050        identity: &BindIdentity,
8051    ) -> Self {
8052        Self {
8053            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8054            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8055            harness: bounded_log_text(&identity.harness),
8056        }
8057    }
8058
8059    /// For a refusal that has no parsed request at hand: names whatever the
8060    /// frame still says, and `unknown` where it says nothing parseable.
8061    fn unknown() -> Self {
8062        Self {
8063            principal: "unknown".to_string(),
8064            scope: "unknown".to_string(),
8065            harness: "unknown".to_string(),
8066        }
8067    }
8068}
8069
8070/// Longest client-chosen text (a scope ref, a harness) a route.open log line
8071/// prints before cutting it.
8072const MAX_LOGGED_TEXT_CHARS: usize = 64;
8073
8074/// `value` Debug-quoted, so control characters land escaped and cannot forge
8075/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
8076/// `…` inside the quotes so one request cannot flood the line.
8077fn bounded_log_text(value: &str) -> String {
8078    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8079    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8080        bounded.push('…');
8081    }
8082    format!("{bounded:?}")
8083}
8084
8085/// The `scope=` field of the `route.open accepted` line, from the stamp that
8086/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
8087/// so a ref holding control characters cannot forge a log line, and cut to 64
8088/// characters with a trailing `…` to bound the line. The bind proof and the
8089/// scope's attributes never appear.
8090fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8091    use subc_protocol::scope::ScopeKind;
8092    let kind = match stamp.kind {
8093        ScopeKind::Head => "head",
8094        ScopeKind::Worker => "worker",
8095        ScopeKind::Ephemeral => "ephemeral",
8096    };
8097    format!(
8098        "{kind}/{}/{}@{}",
8099        principal_label(&stamp.owner),
8100        bounded_log_text(&stamp.scope_ref),
8101        stamp.scope_epoch
8102    )
8103}
8104
8105/// The `scope=` field of a `route.open refused` line for an open refused
8106/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
8107/// The kind is not known yet, so the literal `requested` stands in its place.
8108/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
8109/// owner it has not been checked against the configured modules: a quoted
8110/// owner means the value was never validated. The bind proof never appears.
8111fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8112    let epoch = selector
8113        .scope_epoch
8114        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8115    format!(
8116        "requested/{}/{}@{epoch}",
8117        bounded_log_text(&principal_label(&selector.owner)),
8118        bounded_log_text(&selector.scope_ref),
8119    )
8120}
8121
8122fn principal_label(principal: &Principal) -> String {
8123    match principal {
8124        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8125        Principal::Direct => "direct".to_string(),
8126        other => format!("{other:?}"),
8127    }
8128}
8129
8130fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8131    match request {
8132        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8133        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8134        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8135        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8136        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8137    }
8138}
8139
8140fn parse_client_control_request(
8141    body: &[u8],
8142) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8143    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8144        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8145            Ok(probe)
8146                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8147                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8148            {
8149                ControlRequestBodyError::InvalidBody
8150            }
8151            Ok(_) => ControlRequestBodyError::UnknownOp,
8152            Err(_) => ControlRequestBodyError::InvalidBody,
8153        };
8154        (err, classification)
8155    })
8156}
8157
8158fn parse_module_control_request_from_module(
8159    body: &[u8],
8160) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8161    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8162        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8163            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8164            Ok(_) => ControlRequestBodyError::UnknownOp,
8165            Err(_) => ControlRequestBodyError::InvalidBody,
8166        };
8167        (err, classification)
8168    })
8169}
8170
8171#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8172enum ProviderRoleKind {
8173    ToolProvider,
8174    PipelineStage,
8175    ManagementSurface,
8176    InternalService,
8177}
8178
8179fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8180    match role {
8181        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8182        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8183        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8184        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8185    }
8186}
8187
8188fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8189    roles.iter().map(provider_role_kind).collect()
8190}
8191
8192/// Most refused scope records named individually in the log per sync; the
8193/// `refused` count on the accepted line is always complete.
8194const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8195
8196/// Per-outcome counts of one accepted `scope.sync`, for its log line.
8197#[derive(Debug, Default, PartialEq, Eq)]
8198struct ScopeOutcomeCounts {
8199    created: usize,
8200    replaced: usize,
8201    updated: usize,
8202    unchanged: usize,
8203    refused: usize,
8204}
8205
8206impl ScopeOutcomeCounts {
8207    fn of(results: &[ScopeRecordResult]) -> Self {
8208        let mut counts = Self::default();
8209        for result in results {
8210            let slot = match result.outcome {
8211                ScopeRecordOutcome::Created => &mut counts.created,
8212                ScopeRecordOutcome::Replaced => &mut counts.replaced,
8213                ScopeRecordOutcome::Updated => &mut counts.updated,
8214                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8215                ScopeRecordOutcome::Refused => &mut counts.refused,
8216            };
8217            *slot += 1;
8218        }
8219        counts
8220    }
8221}
8222
8223#[cfg(test)]
8224mod scope_outcome_count_tests {
8225    use super::*;
8226
8227    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8228        ScopeRecordResult {
8229            scope_ref: "r".to_string(),
8230            scope_epoch: 1,
8231            outcome,
8232            code: None,
8233            message: None,
8234            version: None,
8235            parent_state: None,
8236        }
8237    }
8238
8239    /// Each outcome lands in its own count, so a refused record can never be
8240    /// hidden inside the total the log already printed.
8241    #[test]
8242    fn every_outcome_is_counted_in_its_own_field() {
8243        let results = [
8244            result(ScopeRecordOutcome::Created),
8245            result(ScopeRecordOutcome::Created),
8246            result(ScopeRecordOutcome::Replaced),
8247            result(ScopeRecordOutcome::Updated),
8248            result(ScopeRecordOutcome::Unchanged),
8249            result(ScopeRecordOutcome::Refused),
8250            result(ScopeRecordOutcome::Refused),
8251            result(ScopeRecordOutcome::Refused),
8252        ];
8253        assert_eq!(
8254            ScopeOutcomeCounts::of(&results),
8255            ScopeOutcomeCounts {
8256                created: 2,
8257                replaced: 1,
8258                updated: 1,
8259                unchanged: 1,
8260                refused: 3,
8261            }
8262        );
8263    }
8264}
8265
8266/// Return whether a catalog change can create a newly violating live route.
8267/// Removing an attested claim is intentionally excluded: it makes fewer routes
8268/// forbidden and therefore must leave the existing route census untouched.
8269fn capability_census_trigger(
8270    old: Option<&CapabilityDeclarations>,
8271    new: Option<&CapabilityDeclarations>,
8272) -> bool {
8273    let old_provides = old
8274        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8275        .unwrap_or_default();
8276    let old_denies = old
8277        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8278        .unwrap_or_default();
8279    let new = new.cloned().unwrap_or(CapabilityDeclarations {
8280        provides: Vec::new(),
8281        requires: Vec::new(),
8282        must_never_reach: Vec::new(),
8283    });
8284
8285    new.provides
8286        .iter()
8287        .any(|capability| !old_provides.contains(capability))
8288        || new
8289            .must_never_reach
8290            .iter()
8291            .any(|capability| !old_denies.contains(capability))
8292}
8293
8294/// Find the first capability an attested opener denies that an attested target
8295/// claims. Both manifests are live registry records, never cached or client data.
8296fn denied_capability<'a>(
8297    opening_manifest: &'a ModuleManifest,
8298    target_manifest: &ModuleManifest,
8299) -> Option<&'a str> {
8300    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8301    let target_capabilities = target_manifest.capabilities.as_ref()?;
8302    opening_capabilities
8303        .must_never_reach
8304        .iter()
8305        .find(|denied| {
8306            target_capabilities
8307                .provides
8308                .iter()
8309                .any(|provided| provided == *denied)
8310        })
8311        .map(String::as_str)
8312}
8313
8314fn catalog_update_frozen_field_message(
8315    registered: &ModuleManifest,
8316    provides: &[ProviderRole],
8317) -> Option<String> {
8318    let old_has_provides = !registered.provides.is_empty();
8319    let new_has_provides = !provides.is_empty();
8320    if old_has_provides != new_has_provides {
8321        return Some(format!(
8322            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8323            registered.module_id
8324        ));
8325    }
8326
8327    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
8328        return Some(format!(
8329            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8330            registered.module_id
8331        ));
8332    }
8333
8334    let registered_concurrency = manifest_concurrency(registered);
8335    let mut candidate = registered.clone();
8336    candidate.provides = provides.to_vec();
8337    let candidate_concurrency = manifest_concurrency(&candidate);
8338    if candidate_concurrency != registered_concurrency {
8339        return Some(format!(
8340            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8341            registered.module_id, registered_concurrency, candidate_concurrency
8342        ));
8343    }
8344
8345    // control_ops live beside the manifest in the HELLO body, not inside
8346    // ModuleManifest, so a provides-only catalog.update cannot change them.
8347    None
8348}
8349
8350fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8351    manifest.provides.iter().any(is_routable_role)
8352}
8353
8354/// Returns the routable-provider concurrency subc should enforce for this manifest.
8355///
8356/// ToolProvider and ManagementSurface store their delivery concurrency directly.
8357/// InternalService has no role-specific concurrency field, so it retains the
8358/// existing ModuleManaged default for backward compatibility.
8359fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8360    manifest
8361        .provides
8362        .iter()
8363        .find_map(|provider| match provider {
8364            ProviderRole::ToolProvider { concurrency, .. }
8365            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8366            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8367        })
8368        .unwrap_or(Concurrency::ModuleManaged)
8369}
8370
8371/// True when the manifest carries a ManagementSurface role whose concurrency
8372/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
8373/// bytes because the typed manifest deliberately erases that distinction: the
8374/// default exists for wire compatibility, and this probe exists so the default
8375/// stays observable. Any parse irregularity returns false -- the caller only
8376/// logs, and a malformed body already failed registration upstream.
8377fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8378    let has_management_surface = manifest
8379        .provides
8380        .iter()
8381        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8382    if !has_management_surface {
8383        return false;
8384    }
8385    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8386        return false;
8387    };
8388    let Some(provides) = raw
8389        .get("manifest")
8390        .and_then(|manifest| manifest.get("provides"))
8391        .and_then(serde_json::Value::as_array)
8392    else {
8393        return false;
8394    };
8395    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
8396    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
8397    // against the management_surface_manifest_without_concurrency golden, not
8398    // recalled (the externally-tagged guess was this function's first bug).
8399    provides.iter().any(|role| {
8400        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8401            && role.get("concurrency").is_none()
8402    })
8403}
8404
8405fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8406    if peer_version != PROTOCOL_VERSION {
8407        return Err(format!(
8408            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8409        ));
8410    }
8411    Ok(PROTOCOL_VERSION)
8412}
8413
8414fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8415    Frame::build_with_version(
8416        response_version(frame),
8417        FrameType::Pong,
8418        frame.header.flags,
8419        0,
8420        0,
8421        frame.header.corr,
8422        Vec::new(),
8423    )
8424    .map_err(RouterError::FrameBuild)
8425}
8426
8427pub(crate) fn control_error_frame(
8428    frame: &Frame,
8429    code: &'static str,
8430    message: impl Into<String>,
8431) -> Result<Frame, RouterError> {
8432    control_error_body_frame(
8433        frame,
8434        ErrorBody {
8435            code: code.to_string(),
8436            message: message.into(),
8437            detail: None,
8438        },
8439    )
8440}
8441
8442/// A channel-0 ERROR for a refusal the daemon decided itself: every
8443/// route.open refusal, the drain `module_reloading` and scope-change/end
8444/// bodies, spawn-subscription and supervisor refusals. Carries the
8445/// daemon-origin bit, which is what lets a client treat `module_reloading`
8446/// as proof that its request never reached a module. Response, push,
8447/// goodbye and relay-request frames keep the plain control flags in their
8448/// own builders.
8449fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8450    control_error_frame_with_flags(frame, error, daemon_error_flags())
8451}
8452
8453/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8454/// refused route.bind answered as a route.open refusal, or a refused
8455/// module-control RPC. Plain control flags — the code and message are
8456/// module-chosen, so the frame must not claim daemon authorship, even when
8457/// the module picks a daemon code like `module_reloading`.
8458fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8459    control_error_frame_with_flags(frame, error, control_flags())
8460}
8461
8462fn control_error_frame_with_flags(
8463    frame: &Frame,
8464    error: ErrorBody,
8465    flags: Flags,
8466) -> Result<Frame, RouterError> {
8467    let body = serde_json::to_vec(&error).map_err(|err| {
8468        RouterError::backend(
8469            0,
8470            frame.header.corr,
8471            format!("failed to encode control ERROR: {err}"),
8472        )
8473    })?;
8474
8475    Frame::build_with_version(
8476        response_version(frame),
8477        FrameType::Error,
8478        flags,
8479        0,
8480        0,
8481        frame.header.corr,
8482        body,
8483    )
8484    .map_err(RouterError::FrameBuild)
8485}
8486
8487pub(crate) fn control_response_body_frame<T: Serialize>(
8488    frame: &Frame,
8489    reply: &T,
8490    label: &'static str,
8491) -> Result<Frame, RouterError> {
8492    let body = serde_json::to_vec(reply).map_err(|err| {
8493        RouterError::backend(
8494            0,
8495            frame.header.corr,
8496            format!("failed to encode {label}: {err}"),
8497        )
8498    })?;
8499
8500    Frame::build_with_version(
8501        response_version(frame),
8502        FrameType::Response,
8503        control_flags(),
8504        0,
8505        0,
8506        frame.header.corr,
8507        body,
8508    )
8509    .map_err(RouterError::FrameBuild)
8510}
8511
8512/// Map a forwarding failure to the wire code a client sees.
8513///
8514/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8515/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8516/// chosen here decides whether a caller retries or gives up.
8517///
8518/// That makes attribution the load-bearing property, not merely having a code. A
8519/// permanent fault published as a retryable one produces a fleet-wide retry storm
8520/// against something that can never recover; a transient fault published as
8521/// permanent gives up on work that would have succeeded. Both look correct in a
8522/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8523/// the mapping per variant rather than merely asserting that some code exists.
8524///
8525/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8526/// two codes on the same side of the boundary passes it. Measured rather than
8527/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8528/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8529/// a test named for something else that happens to assert the string.
8530///
8531/// That accidental coverage is deliberately left alone rather than promoted to a
8532/// named test, because it guards a property this function does not promise.
8533/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8534/// specific code within a class, so identity is free to change and only the
8535/// partition is a contract. Splitting it out would assert a guarantee nothing
8536/// depends on — and a suite that promises more than the code does is the harder
8537/// thing to correct later, because the next reader cannot tell which assertions
8538/// are load-bearing.
8539///
8540/// Pin identity here the moment a consumer branches on a specific code.
8541fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8542    match err {
8543        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8544        ForwardingError::NoModuleConnection => "target_unavailable",
8545        ForwardingError::ModuleReloading { .. } => "module_reloading",
8546        ForwardingError::ClientRouteChannelExhausted { .. }
8547        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8548        ForwardingError::StaleModuleEndpoint
8549        | ForwardingError::UnknownReservation { .. }
8550        | ForwardingError::ConnectionClosing { .. }
8551        | ForwardingError::ClientEgressClosed { .. }
8552        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8553        // Only a swap candidate's registration can produce this, and it means
8554        // exactly what a second active HELLO for a live id means.
8555        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8556        ForwardingError::RelayCorrelationExhausted
8557        | ForwardingError::RouteOpenBuild(_)
8558        | ForwardingError::Poisoned => "forwarding_error",
8559    }
8560}
8561
8562fn response_version(frame: &Frame) -> u8 {
8563    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8564        frame.header.ver
8565    } else {
8566        PROTOCOL_VERSION
8567    }
8568}
8569
8570fn control_flags() -> Flags {
8571    Flags::new(false, Priority::Passive, false)
8572}
8573
8574/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8575/// channel. The target is the module (a client never saw the route), so this
8576/// takes the module path: delivered late rather than dropped when the module's
8577/// queue is momentarily full, and never closing its connection.
8578fn send_goodbye_target_best_effort(
8579    counters: &DaemonCounters,
8580    target: &GoodbyeTarget,
8581    context: &'static str,
8582) {
8583    let Ok(frame) = Frame::build_with_version(
8584        target.negotiated_ver,
8585        FrameType::Goodbye,
8586        control_flags(),
8587        target.channel,
8588        target.epoch,
8589        0,
8590        Vec::new(),
8591    ) else {
8592        return;
8593    };
8594    crate::forwarding::send_module_route_goodbye(
8595        counters,
8596        &target.sink,
8597        frame,
8598        target.module_id.as_deref(),
8599        context,
8600    );
8601}
8602
8603pub(crate) fn send_route_control_pushes(
8604    forwarding: &ForwardingTable,
8605    routes: Vec<EndpointRoute>,
8606    push: ClientControlPush,
8607) {
8608    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8609    for route in routes {
8610        let target = route.goodbye_target;
8611        if let Some((existing, channels)) = targets
8612            .iter_mut()
8613            .find(|(existing, _)| existing.connection_id == target.connection_id)
8614        {
8615            debug_assert_eq!(
8616                existing.negotiated_ver, target.negotiated_ver,
8617                "one connection cannot negotiate multiple frame versions"
8618            );
8619            if !channels.contains(&target.channel) {
8620                channels.push(target.channel);
8621            }
8622            continue;
8623        }
8624        let channel = target.channel;
8625        targets.push((target, vec![channel]));
8626    }
8627    for (target, mut channels) in targets {
8628        channels.sort_unstable();
8629        let mut push = push.clone();
8630        match &mut push {
8631            ClientControlPush::RouteClosing {
8632                channels: covered, ..
8633            }
8634            | ClientControlPush::RouteClosed {
8635                channels: covered, ..
8636            } => *covered = channels,
8637        }
8638        let body = match serde_json::to_vec(&push) {
8639            Ok(body) => body,
8640            Err(err) => {
8641                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8642                continue;
8643            }
8644        };
8645        let frame = match Frame::build_with_version(
8646            target.negotiated_ver,
8647            FrameType::Push,
8648            control_flags(),
8649            0,
8650            0,
8651            0,
8652            body.clone(),
8653        ) {
8654            Ok(frame) => frame,
8655            Err(err) => {
8656                warn!(
8657                    route_channel = target.channel,
8658                    error = %err,
8659                    "failed to build route lifecycle control PUSH frame"
8660                );
8661                continue;
8662            }
8663        };
8664        if let Err(err) = target.sink.try_send(frame) {
8665            if target.close_on_delivery_failure() {
8666                warn!(
8667                    target_connection_id = target.connection_id.get(),
8668                    route_channel = target.channel,
8669                    error = %err,
8670                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8671                );
8672                let _ = forwarding.escalate_client_delivery_failure(
8673                    target.connection_id,
8674                    target.channel,
8675                    target.epoch,
8676                    CloseReason::new(
8677                        "route_lifecycle_push_delivery_failed",
8678                        format!(
8679                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8680                            target.channel
8681                        ),
8682                    ),
8683                    crate::forwarding::UndeliveredFrame {
8684                        module_id: target.module_id.as_deref(),
8685                        sink: &target.sink,
8686                    },
8687                );
8688            }
8689        }
8690    }
8691}
8692
8693#[cfg(test)]
8694mod tests {
8695    #[tokio::test]
8696    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8697        let registry = Arc::new(Registry::default());
8698        let handler = ControlHandler::new(registry);
8699        assert_eq!(
8700            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8701            None
8702        );
8703        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8704        hello_via_sink(
8705            &handler,
8706            &ctx,
8707            &mut rx,
8708            hello_frame("daemon", PROTOCOL_VERSION, 1),
8709        )
8710        .await;
8711        assert_eq!(
8712            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8713            Some("daemon".into())
8714        );
8715        assert_eq!(
8716            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8717            None
8718        );
8719        assert_eq!(
8720            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8721            None,
8722            "a registered module is not necessarily supervised"
8723        );
8724        assert_eq!(
8725            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8726            None,
8727            "aggregate provenance has no single target"
8728        );
8729        assert_eq!(
8730            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8731            None
8732        );
8733    }
8734
8735    mod peer_manifest {
8736        include!("peer_manifest_tests.rs");
8737    }
8738    mod activity_read {
8739        include!("activity_read_tests.rs");
8740    }
8741
8742    mod lifecycle_activity {
8743        include!("control_lifecycle_tests.rs");
8744    }
8745    #[cfg(unix)]
8746    #[tokio::test]
8747    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8748        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8749        let path = dir.join("subc.jsonc");
8750        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8751            "program":"/bin/sleep","args":["60"],"protocol":"none",
8752            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8753        }}}).to_string()).unwrap();
8754        let mut configured = crate::daemon_config::load(&path)
8755            .unwrap()
8756            .unwrap()
8757            .modules
8758            .pop()
8759            .unwrap();
8760        let registry = std::sync::Arc::new(crate::Registry::default());
8761        let handle = crate::SupervisorHandle::new();
8762        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8763            .with_handle(handle.clone());
8764        let module = supervisor
8765            .supervise_configured_with_health(
8766                configured.module_spec(),
8767                true,
8768                configured.health.clone(),
8769                None,
8770                configured.restart,
8771            )
8772            .unwrap();
8773        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8774        let before = module.status().unwrap().pid;
8775        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8776        configured.health.cadence = std::time::Duration::from_secs(3600);
8777        let health_only = handler
8778            .reconcile_supervised_modules(
8779                &supervisor,
8780                vec![configured.clone()],
8781                false,
8782                Principal::Direct,
8783            )
8784            .await
8785            .unwrap();
8786        assert!(
8787            health_only.0.changed_pending_reload.is_empty(),
8788            "health policy is already applied live"
8789        );
8790        assert_eq!(module.status().unwrap().pid, before);
8791        assert_eq!(
8792            module.configuration().unwrap().1.http,
8793            configured.health.http
8794        );
8795        configured.args = vec!["61".into()];
8796        let launch = handler
8797            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8798            .await
8799            .unwrap();
8800        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8801        assert_eq!(
8802            module.status().unwrap().pid,
8803            before,
8804            "a launch edit is stored until reload"
8805        );
8806        module.drain().await.unwrap();
8807    }
8808    use cortexkit_test_support::ScratchDir;
8809    use std::{
8810        collections::BTreeMap,
8811        fmt,
8812        path::PathBuf,
8813        sync::{Arc, Mutex},
8814        time::Duration,
8815    };
8816
8817    use serde_json::{json, Value};
8818    use subc_protocol::{
8819        manifest::{
8820            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8821            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8822        },
8823        session::HealthStatus,
8824        FrameType,
8825    };
8826
8827    use super::*;
8828    use crate::{
8829        forwarding::{DataRoute, DataRouteState},
8830        registry::ChannelState,
8831        router::FrameSink,
8832        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8833        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8834        RouteCtx, Router,
8835    };
8836    use tokio::{
8837        sync::mpsc,
8838        time::{sleep, Instant},
8839    };
8840    use tracing::{
8841        field::{Field, Visit},
8842        Event, Subscriber,
8843    };
8844    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8845
8846    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8847    ///
8848    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8849    /// is only populated for `tests/*.rs` integration test binaries -- this file
8850    /// compiles as part of the library target, which gets neither. This test's
8851    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8852    /// and the sibling binary lives two directories up at
8853    /// `<target-dir>/<profile>/fake-aft-stub`.
8854    ///
8855    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8856    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8857    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8858    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8859    /// `NotFound`, which reads as a broken test rather than an unbuilt
8860    /// dependency -- so state the cause and the remedy instead. Deliberately a
8861    /// panic and not a silent skip: a test that quietly passes when it could not
8862    /// run is worse than one that fails, because it reports health it never
8863    /// verified.
8864    fn fake_aft_stub_path() -> PathBuf {
8865        let mut path = std::env::current_exe().expect("current_exe available in tests");
8866        path.pop(); // .../deps/
8867        path.pop(); // .../<profile>/
8868        path.push(if cfg!(windows) {
8869            "fake-aft-stub.exe"
8870        } else {
8871            "fake-aft-stub"
8872        });
8873        assert!(
8874            path.exists(),
8875            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8876             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8877            path.display()
8878        );
8879        path
8880    }
8881
8882    /// Whether clients retry `code` in place: the predicate itself, never a copy
8883    /// of its set. A copied list breaks silently when a code is added to or
8884    /// removed from the real one, and a stale copy here would let exactly the
8885    /// failure this test exists to catch pass.
8886    fn client_retries(code: &str) -> bool {
8887        subc_protocol::error_codes::is_retryable_route_open(code)
8888    }
8889
8890    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8891    /// of failure is worse than publishing none. A permanent fault dressed as
8892    /// retryable makes every client in the fleet retry forever against something
8893    /// that cannot recover; a transient fault dressed as permanent abandons work
8894    /// that would have succeeded.
8895    ///
8896    /// Asserting "a code exists" cannot catch either, because the string is free
8897    /// to say anything. This enumerates every variant and pins which side of the
8898    /// retry boundary it lands on, so a new variant must be classified here
8899    /// deliberately rather than inheriting whichever arm it was appended to.
8900    #[test]
8901    fn retryability_of_forwarding_codes_matches_the_failure() {
8902        // Transient by nature: the target is booting, reloading, or its endpoint
8903        // was swapped mid-flight. Retrying is how these resolve.
8904        let transient = [
8905            ForwardingError::NoModuleConnection,
8906            ForwardingError::ModuleReloading {
8907                module_id: "m".into(),
8908            },
8909            ForwardingError::StaleModuleEndpoint,
8910            ForwardingError::UnknownReservation {
8911                client_channel: 1,
8912                module_channel: 1,
8913            },
8914            ForwardingError::ConnectionClosing {
8915                connection_id: ConnectionId::new(1),
8916            },
8917            ForwardingError::ClientEgressClosed {
8918                connection_id: ConnectionId::new(1),
8919            },
8920            ForwardingError::ModuleEgressUnavailable {
8921                connection_id: ConnectionId::new(1),
8922            },
8923        ];
8924        for err in transient {
8925            let code = forwarding_error_code(&err);
8926            assert!(
8927                client_retries(code),
8928                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8929            );
8930        }
8931
8932        // Not fixed by retrying. Channel and correlation exhaustion need the
8933        // caller to close routes, and a poisoned lock is a daemon that cannot
8934        // recover at all — the worst thing to advertise as retryable, since every
8935        // client would storm a daemon that will never answer.
8936        let permanent = [
8937            ForwardingError::ConnectionRoleConflict {
8938                connection_id: ConnectionId::new(1),
8939            },
8940            ForwardingError::ClientRouteChannelExhausted {
8941                connection_id: ConnectionId::new(1),
8942            },
8943            ForwardingError::ModuleRouteChannelExhausted {
8944                endpoint: ModuleEndpointId {
8945                    connection_id: ConnectionId::new(1),
8946                    generation: 1,
8947                },
8948            },
8949            ForwardingError::RelayCorrelationExhausted,
8950            ForwardingError::RouteOpenBuild("x".into()),
8951            ForwardingError::Poisoned,
8952        ];
8953        for err in permanent {
8954            let code = forwarding_error_code(&err);
8955            assert!(
8956                !client_retries(code),
8957                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8958            );
8959        }
8960    }
8961
8962    /// The principal is the daemon's answer to "who is calling", and modules
8963    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8964    /// plexus gates connector invocation. So a stamp is an authorization input in
8965    /// another process, not a label — and both possible answers SUCCEED, which is
8966    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8967    /// first-party capability to something that never proved it; a supervised one
8968    /// stamped `Direct` silently strips a module of capability it is entitled to.
8969    ///
8970    /// Neither shows up in a test that only checks the bind succeeded. Before this
8971    /// test the only coverage was accidental —
8972    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8973    /// stamped principal on its way past, so narrowing that wire-shape test to its
8974    /// stated subject would have deleted the last assertion on this value. It
8975    /// still asserts the stamp, which is now redundancy rather than the only
8976    /// guard: both fail under the same mutation, and this one names the reason.
8977    /// SCOPE: this handler's supervisor has spawned nothing, so
8978    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8979    /// is unreachable here. Both assertions below are refusals, and a mutant that
8980    /// refuses everything would satisfy them.
8981    ///
8982    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8983    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8984    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8985    /// at source rather than assumed, since a citation is a claim about another
8986    /// file and ages like one. Recorded because a harness that structurally
8987    /// cannot reach an arm reports "none" for that arm identically to one that
8988    /// covers it and found nothing.
8989    #[tokio::test]
8990    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8991        let handler = ControlHandler::default();
8992        let frame =
8993            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8994
8995        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8996        // any process holding the connection file. Nothing was proved, so nothing
8997        // may be granted beyond the unattested floor.
8998        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8999        assert_eq!(
9000            stamped,
9001            Principal::Direct,
9002            "a caller that proved nothing must not be stamped as a supervised module"
9003        );
9004
9005        // A claimed module_id with a nonce no supervised child was given is a
9006        // forgery attempt, not a weaker caller: it must be REFUSED rather than
9007        // quietly demoted to Direct, or an impersonation attempt looks identical
9008        // to an ordinary unattested connection.
9009        let forged = handler
9010            .route_open_principal(
9011                &frame,
9012                Some(ConsumerIdentity {
9013                    module_id: "aft".to_string(),
9014                    launch_nonce: "not-a-real-nonce".to_string(),
9015                }),
9016            )
9017            .unwrap();
9018        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9019        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9020    }
9021
9022    /// The test above hands `route_open_principal` an identity it built itself,
9023    /// which proves the stamping rule and nothing about where the identity comes
9024    /// from. The real producer is a wire body, and the two are joined by a serde
9025    /// field name that nothing else asserts.
9026    ///
9027    /// That join fails quietly in one specific way: an unrecognised key is simply
9028    /// absent after parsing, so a renamed or misspelled `consumer_identity`
9029    /// yields `None` and every supervised module silently drops to `Direct`.
9030    /// Capability-wise that is the safe direction, but it surfaces far from its
9031    /// cause — as a module mysteriously refused bash — and it would pass every
9032    /// test that builds its own input.
9033    ///
9034    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
9035    /// would break every client the moment the daemon gains a field, trading a
9036    /// quiet demotion for a hard refusal on additive change. Asserting the join
9037    /// instead means a rename breaks a test here rather than the fleet.
9038    #[test]
9039    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9040        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9041        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9042        let ClientControlRequest::RouteOpen {
9043            consumer_identity, ..
9044        } = parsed
9045        else {
9046            panic!("route.open body must parse as RouteOpen");
9047        };
9048        assert_eq!(
9049            consumer_identity,
9050            Some(ConsumerIdentity {
9051                module_id: "aft".to_string(),
9052                launch_nonce: "n".to_string(),
9053            }),
9054            "the wire field name must reach the value route_open_principal reads"
9055        );
9056    }
9057
9058    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9059        ModuleManifest::builder(module_id, "0.1.0")
9060            .protocol_ver(protocol_ver)
9061            .provides(vec![ProviderRole::ToolProvider {
9062                tools: vec![Tool {
9063                    name: "read".to_string(),
9064                    description: None,
9065                    execution_mode: ExecutionMode::Pure,
9066                    schema: json!({"type": "object"}),
9067                }],
9068                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9069                concurrency: Concurrency::ModuleManaged,
9070                emits_push: true,
9071                sub_supervises: true,
9072            }])
9073            .build()
9074    }
9075
9076    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9077        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9078    }
9079
9080    fn hello_frame_with_control_ops(
9081        module_id: &str,
9082        protocol_ver: u8,
9083        corr: u64,
9084        control_ops: Option<Vec<String>>,
9085    ) -> Frame {
9086        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9087    }
9088
9089    fn hello_frame_with_nonce(
9090        module_id: &str,
9091        protocol_ver: u8,
9092        corr: u64,
9093        launch_nonce: Option<&str>,
9094    ) -> Frame {
9095        hello_frame_full(
9096            module_id,
9097            protocol_ver,
9098            corr,
9099            None,
9100            launch_nonce.map(ToOwned::to_owned),
9101        )
9102    }
9103
9104    fn hello_frame_full(
9105        module_id: &str,
9106        protocol_ver: u8,
9107        corr: u64,
9108        control_ops: Option<Vec<String>>,
9109        launch_nonce: Option<String>,
9110    ) -> Frame {
9111        let body = serde_json::to_vec(&ModuleHelloBody {
9112            manifest: manifest(module_id, protocol_ver),
9113            protocol_ver,
9114            control_ops,
9115            launch_nonce,
9116        })
9117        .unwrap();
9118        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9119    }
9120
9121    fn hello_frame_with_rich_ui(
9122        module_id: &str,
9123        protocol_ver: u8,
9124        corr: u64,
9125        launch_nonce: Option<&str>,
9126        rich_ui: Value,
9127    ) -> Frame {
9128        let mut body = serde_json::to_value(ModuleHelloBody {
9129            manifest: manifest(module_id, protocol_ver),
9130            protocol_ver,
9131            control_ops: None,
9132            launch_nonce: launch_nonce.map(str::to_string),
9133        })
9134        .unwrap();
9135        body["manifest"]["rich_ui"] = rich_ui;
9136        Frame::build(
9137            FrameType::Hello,
9138            control_flags(),
9139            0,
9140            0,
9141            corr,
9142            serde_json::to_vec(&body).unwrap(),
9143        )
9144        .unwrap()
9145    }
9146
9147    fn non_routable_hello_frame_with_control_ops(
9148        module_id: &str,
9149        corr: u64,
9150        control_ops: Option<Vec<String>>,
9151    ) -> Frame {
9152        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9153        manifest.provides.clear();
9154        let body = serde_json::to_vec(&ModuleHelloBody {
9155            manifest,
9156            protocol_ver: PROTOCOL_VERSION,
9157            control_ops,
9158            launch_nonce: None,
9159        })
9160        .unwrap();
9161        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9162    }
9163
9164    fn capability_grammar_hello_frame(
9165        capabilities: Value,
9166        runtime_computed: Option<Value>,
9167        corr: u64,
9168    ) -> Frame {
9169        let mut body = serde_json::to_value(ModuleHelloBody {
9170            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9171            protocol_ver: PROTOCOL_VERSION,
9172            control_ops: None,
9173            launch_nonce: None,
9174        })
9175        .expect("HELLO body serializes");
9176        body["manifest"]["capabilities"] = capabilities;
9177        if let Some(runtime_computed) = runtime_computed {
9178            body["runtime_computed"] = runtime_computed;
9179        }
9180        Frame::build(
9181            FrameType::Hello,
9182            control_flags(),
9183            0,
9184            0,
9185            corr,
9186            serde_json::to_vec(&body).expect("HELLO body reserializes"),
9187        )
9188        .expect("HELLO frame builds")
9189    }
9190
9191    fn channel_request(channel: u16, corr: u64) -> Frame {
9192        Frame::build(
9193            FrameType::Request,
9194            Flags::new(true, Priority::Interactive, false),
9195            channel,
9196            0,
9197            corr,
9198            b"opaque".to_vec(),
9199        )
9200        .unwrap()
9201    }
9202
9203    fn route_ctx(
9204        connection_id: ConnectionId,
9205    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9206        let (tx, rx) = mpsc::channel(8);
9207        (
9208            RouteCtx {
9209                connection_id,
9210                egress: FrameSink::new(tx),
9211            },
9212            rx,
9213        )
9214    }
9215
9216    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9217        serde_json::from_slice(&frame.body).unwrap()
9218    }
9219
9220    /// Register a module over a connection that has a sink and return the
9221    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
9222    /// module's own sink rather than returning it as a reply, so the ack is
9223    /// taken off `rx` here and whatever the test reads next is what followed it.
9224    async fn hello_via_sink(
9225        handler: &ControlHandler,
9226        ctx: &RouteCtx,
9227        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9228        hello: Frame,
9229    ) -> Frame {
9230        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9231        assert!(
9232            replies.is_empty(),
9233            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9234        );
9235        let ack = rx
9236            .try_recv()
9237            .expect("HELLO_ACK is queued on the module sink")
9238            .frame;
9239        assert_eq!(ack.header.ty, FrameType::HelloAck);
9240        ack
9241    }
9242
9243    fn parse_error(frame: &Frame) -> Value {
9244        serde_json::from_slice(&frame.body).unwrap()
9245    }
9246
9247    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9248        serde_json::from_slice(&frame.body).unwrap()
9249    }
9250
9251    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9252        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9253            route_channel,
9254            route_epoch: 0,
9255            kind,
9256        })
9257        .unwrap();
9258        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9259    }
9260
9261    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9262        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9263            module_id: module_id.to_string(),
9264        })
9265        .unwrap();
9266        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9267    }
9268
9269    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9270        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9271    }
9272
9273    // Exercise retired opener fields without putting them back in BindIdentity.
9274    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9275        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9276        let mut frame = route_open_frame(corr, module_id, root);
9277        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9278        body["identity"]["project_root"] = serde_json::json!(path.as_path());
9279        frame.body = serde_json::to_vec(&body).unwrap();
9280        frame
9281    }
9282
9283    fn route_open_frame_with_consumer_capabilities(
9284        corr: u64,
9285        module_id: &str,
9286        _project_root: ScratchDir,
9287        consumer_capabilities: Option<Vec<String>>,
9288    ) -> Frame {
9289        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9290            busy_refusals: None,
9291            target: RouteTarget::ToolProvider {
9292                module_id: module_id.to_string(),
9293            },
9294            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9295            consumer_identity: None,
9296            consumer_capabilities,
9297            role_versions: None,
9298            admission_facts: None,
9299            scope: None,
9300        })
9301        .unwrap();
9302        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9303    }
9304
9305    fn route_open_frame_with_role_versions(
9306        corr: u64,
9307        module_id: &str,
9308        _project_root: ScratchDir,
9309        role_versions: Option<BTreeMap<String, String>>,
9310    ) -> Frame {
9311        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9312            busy_refusals: None,
9313            target: RouteTarget::ToolProvider {
9314                module_id: module_id.to_string(),
9315            },
9316            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9317            consumer_identity: None,
9318            consumer_capabilities: None,
9319            role_versions,
9320            admission_facts: None,
9321            scope: None,
9322        })
9323        .unwrap();
9324        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9325    }
9326
9327    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9328        entries
9329            .iter()
9330            .map(|(role, version)| (role.to_string(), version.to_string()))
9331            .collect()
9332    }
9333
9334    fn route_open_frame_with_admission_facts(
9335        corr: u64,
9336        module_id: &str,
9337        _project_root: ScratchDir,
9338        consumer_identity: Option<subc_control::ConsumerIdentity>,
9339        facts: Option<Value>,
9340    ) -> Frame {
9341        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9342            busy_refusals: None,
9343            target: RouteTarget::ToolProvider {
9344                module_id: module_id.to_string(),
9345            },
9346            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9347            consumer_identity,
9348            consumer_capabilities: None,
9349            role_versions: None,
9350            admission_facts: facts,
9351            scope: None,
9352        })
9353        .unwrap();
9354        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9355    }
9356
9357    #[derive(Clone, Default)]
9358    struct EventCapture {
9359        events: Arc<Mutex<Vec<CapturedEvent>>>,
9360    }
9361
9362    #[derive(Clone, Debug)]
9363    struct CapturedEvent {
9364        target: String,
9365        level: tracing::Level,
9366        fields: BTreeMap<String, String>,
9367    }
9368
9369    impl EventCapture {
9370        fn events(&self) -> Vec<CapturedEvent> {
9371            self.events.lock().unwrap().clone()
9372        }
9373    }
9374
9375    impl<S> Layer<S> for EventCapture
9376    where
9377        S: Subscriber,
9378    {
9379        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9380            let mut visitor = EventFieldVisitor::default();
9381            event.record(&mut visitor);
9382            self.events.lock().unwrap().push(CapturedEvent {
9383                target: event.metadata().target().to_string(),
9384                level: *event.metadata().level(),
9385                fields: visitor.fields,
9386            });
9387        }
9388    }
9389
9390    #[derive(Default)]
9391    struct EventFieldVisitor {
9392        fields: BTreeMap<String, String>,
9393    }
9394
9395    impl Visit for EventFieldVisitor {
9396        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9397            self.fields
9398                .insert(field.name().to_string(), format!("{value:?}"));
9399        }
9400    }
9401
9402    fn health_response(corr: u64, status: HealthStatus) -> Frame {
9403        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9404            status,
9405            detail: Some("warming".to_string()),
9406            metrics: Some(json!({"queue_depth": 3})),
9407        })
9408        .unwrap();
9409        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9410    }
9411
9412    fn route_bind_ack(corr: u64) -> Frame {
9413        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9414        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9415    }
9416
9417    fn unique_project_root(label: &str) -> ScratchDir {
9418        ScratchDir::new(label)
9419    }
9420
9421    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9422        match parse_route_poll(frame) {
9423            ClientControlResponse::RoutePoll {
9424                status: None,
9425                live: Some(live),
9426                ..
9427            } => assert_eq!(live, expected_live),
9428            other => panic!("unexpected route.poll response: {other:?}"),
9429        }
9430    }
9431
9432    fn bind_liveness_route(
9433        registry: &Registry,
9434        forwarding: &ForwardingTable,
9435        module_id: &str,
9436    ) -> (RouteCtx, u16, u32) {
9437        let module_connection = ConnectionId::new(101);
9438        let client_connection = ConnectionId::new(202);
9439        let registration = registry
9440            .register_with_control_ops(
9441                manifest(module_id, PROTOCOL_VERSION),
9442                PROTOCOL_VERSION,
9443                module_connection,
9444                module_baseline_control_ops(),
9445            )
9446            .unwrap();
9447        let (module_tx, _module_rx) = mpsc::channel(8);
9448        let endpoint = forwarding
9449            .register_module_connection(
9450                module_connection,
9451                module_id.to_string(),
9452                PROTOCOL_VERSION,
9453                manifest_concurrency(&registration.manifest),
9454                FrameSink::new(module_tx),
9455            )
9456            .unwrap();
9457        let (client_ctx, _client_rx) = route_ctx(client_connection);
9458        let pending = forwarding
9459            .begin_route_bind_relay_for_test(
9460                client_connection,
9461                client_ctx.egress.clone(),
9462                1,
9463                module_id,
9464            )
9465            .unwrap();
9466        assert_eq!(pending.endpoint, endpoint);
9467        let route_channel = pending.client_channel;
9468        let route_epoch = pending.client_epoch;
9469        forwarding
9470            .complete_pending_relay(
9471                module_connection,
9472                pending.corr,
9473                RouteBindRelayOutcome::Accepted,
9474            )
9475            .unwrap();
9476        (client_ctx, route_channel, route_epoch)
9477    }
9478
9479    struct FakeProcessLiveness {
9480        live: Option<bool>,
9481    }
9482
9483    impl ModuleProcessLiveness for FakeProcessLiveness {
9484        fn process_live(&self, _module_id: &str) -> Option<bool> {
9485            self.live
9486        }
9487    }
9488
9489    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9490    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9491    {
9492        let registry = Arc::new(Registry::default());
9493        let supervisor_handle = SupervisorHandle::new();
9494        let supervisor = Supervisor::new_for_test(
9495            Arc::clone(&registry),
9496            RestartPolicy::new(1, Duration::from_millis(10)),
9497        )
9498        .with_handle(supervisor_handle.clone());
9499        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9500        let module = supervisor
9501            .spawn(ModuleSpec {
9502                carry_containment: false,
9503                module_id: "stderr-tail-wire".to_string(),
9504                program: fake_aft_stub_path(),
9505                args: Vec::new(),
9506                env: vec![
9507                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9508                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9509                ],
9510                reserved: false,
9511                reserved_prefixes: Vec::new(),
9512                protocol: ModuleProtocol::Subc,
9513                overlap: Default::default(),
9514            })
9515            .unwrap();
9516
9517        let deadline = Instant::now() + Duration::from_secs(5);
9518        loop {
9519            let tail = module.stderr_tail(None, None);
9520            if tail
9521                .entries
9522                .iter()
9523                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9524                && tail.entries.iter().any(|entry| {
9525                    matches!(
9526                        entry,
9527                        TailEntry::Line {
9528                            truncated: true,
9529                            ..
9530                        }
9531                    )
9532                })
9533            {
9534                break;
9535            }
9536            assert!(
9537                Instant::now() < deadline,
9538                "module did not produce a truncated line and restart boundary: {tail:?}"
9539            );
9540            sleep(Duration::from_millis(10)).await;
9541        }
9542
9543        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9544        let request = ClientControlRequest::SupervisorStderrTail {
9545            module_id: "stderr-tail-wire".to_string(),
9546            max_lines: None,
9547            max_bytes: None,
9548        };
9549        let frame = Frame::build(
9550            FrameType::Request,
9551            control_flags(),
9552            0,
9553            0,
9554            1,
9555            serde_json::to_vec(&request).unwrap(),
9556        )
9557        .unwrap();
9558        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9559        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9560        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9561            serde_json::from_slice(&responses[0].body).unwrap()
9562        else {
9563            panic!("expected supervisor.stderr_tail response");
9564        };
9565
9566        assert!(
9567            tail.entries
9568                .iter()
9569                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9570            "the control response lost the restart boundary"
9571        );
9572        let Some(StderrTailEntry::Line {
9573            text,
9574            truncated,
9575            at_ms,
9576        }) = tail.entries.iter().find(|entry| {
9577            matches!(
9578                entry,
9579                StderrTailEntry::Line {
9580                    truncated: true,
9581                    ..
9582                }
9583            )
9584        })
9585        else {
9586            panic!("the control response lost the truncated line");
9587        };
9588        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9589        assert!(*truncated);
9590        assert!(
9591            at_ms.is_some(),
9592            "the control response lost the line's capture time"
9593        );
9594    }
9595
9596    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9597    /// read done on the worker thread would stall every other task until it
9598    /// finished; the read must run off the worker so this test's own task keeps
9599    /// running while the read is paused.
9600    #[tokio::test(flavor = "current_thread")]
9601    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9602        let dir = ScratchDir::new("terminals-off-worker");
9603        let journal_path = dir.join("terminals.jsonl");
9604        let registry = Arc::new(Registry::default());
9605        let supervisor_handle = SupervisorHandle::new();
9606        let supervisor =
9607            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9608                .with_handle(supervisor_handle.clone())
9609                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9610        let module = supervisor
9611            .spawn(ModuleSpec {
9612                carry_containment: false,
9613                module_id: "terminal-off-worker".to_string(),
9614                program: fake_aft_stub_path(),
9615                args: Vec::new(),
9616                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9617                reserved: false,
9618                reserved_prefixes: Vec::new(),
9619                protocol: ModuleProtocol::Subc,
9620                overlap: Default::default(),
9621            })
9622            .unwrap();
9623        let deadline = Instant::now() + Duration::from_secs(5);
9624        while module.terminal_history().entries.len() != 2 {
9625            assert!(Instant::now() < deadline, "module did not record two exits");
9626            sleep(Duration::from_millis(10)).await;
9627        }
9628
9629        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9630        let handler =
9631            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9632        let frame = Frame::build(
9633            FrameType::Request,
9634            control_flags(),
9635            0,
9636            0,
9637            1,
9638            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9639                module_id: "terminal-off-worker".to_string(),
9640            })
9641            .unwrap(),
9642        )
9643        .unwrap();
9644        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9645        let spawned_at = std::time::Instant::now();
9646        let read = tokio::spawn({
9647            let handler = Arc::clone(&handler);
9648            async move { handler.handle_control_frame(&ctx, frame).await }
9649        });
9650        // Waiting for the pause from a blocking thread keeps this task pending,
9651        // so the runtime's single worker is free to run the read task.
9652        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9653            .await
9654            .unwrap()
9655            .expect("the history read reached its pause");
9656        let elapsed = spawned_at.elapsed();
9657        assert!(
9658            elapsed < Duration::from_secs(2) && !read.is_finished(),
9659            "this task could not run while the history read was paused \
9660             (resumed after {elapsed:?}, read finished: {})",
9661            read.is_finished()
9662        );
9663
9664        drop(release);
9665        let responses = read.await.unwrap().unwrap();
9666        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9667        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9668            panic!("expected supervisor.terminals response");
9669        };
9670        assert_eq!(terminals.entries.len(), 2);
9671        assert_eq!(terminals.journal_skipped_lines, 0);
9672        assert_eq!(terminals.journal_read_errors, 0);
9673    }
9674
9675    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9676    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9677        let registry = Arc::new(Registry::default());
9678        let supervisor_handle = SupervisorHandle::new();
9679        let supervisor =
9680            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9681                .with_handle(supervisor_handle.clone());
9682        let module = supervisor
9683            .spawn(ModuleSpec {
9684                carry_containment: false,
9685                module_id: "terminal-golden".to_string(),
9686                program: fake_aft_stub_path(),
9687                args: Vec::new(),
9688                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9689                reserved: false,
9690                reserved_prefixes: Vec::new(),
9691                protocol: ModuleProtocol::Subc,
9692                overlap: Default::default(),
9693            })
9694            .unwrap();
9695
9696        let deadline = Instant::now() + Duration::from_secs(5);
9697        while module.terminal_history().entries.len() != 2 {
9698            assert!(
9699                Instant::now() < deadline,
9700                "module did not retain two terminal exits: {:?}",
9701                module.terminal_history()
9702            );
9703            sleep(Duration::from_millis(10)).await;
9704        }
9705
9706        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9707        let request = ClientControlRequest::SupervisorTerminals {
9708            module_id: "terminal-golden".to_string(),
9709        };
9710        let frame = Frame::build(
9711            FrameType::Request,
9712            control_flags(),
9713            0,
9714            0,
9715            1,
9716            serde_json::to_vec(&request).unwrap(),
9717        )
9718        .unwrap();
9719        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9720        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9721        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9722        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9723            panic!("expected supervisor.terminals response");
9724        };
9725        assert_eq!(terminals.entries.len(), 2);
9726        assert_eq!(terminals.dropped, 0);
9727
9728        let mut rendered = serde_json::to_value(response).unwrap();
9729        // Wall-clock fields are the observation contract, but not stable fixture
9730        // bytes; normalize only them after the real handler has shaped the response.
9731        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9732        for (index, entry) in rendered["entries"]
9733            .as_array_mut()
9734            .expect("terminal response entries array")
9735            .iter_mut()
9736            .enumerate()
9737        {
9738            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9739        }
9740
9741        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9742            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9743        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9744        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9745            std::fs::write(&golden_path, &serialized).unwrap();
9746        }
9747        let expected: Value =
9748            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9749        assert_eq!(rendered, expected);
9750    }
9751
9752    #[test]
9753    fn hello_registers_manifest_and_returns_ack() {
9754        let registry = Arc::new(Registry::default());
9755        let handler = ControlHandler::new(Arc::clone(&registry));
9756        let conn = ConnectionId::new(1);
9757
9758        let responses = handler
9759            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9760            .unwrap();
9761
9762        assert_eq!(responses.len(), 1);
9763        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9764        assert_eq!(responses[0].header.channel, 0);
9765        assert_eq!(responses[0].header.corr, 7);
9766        let ack = parse_ack(&responses[0]);
9767        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9768        assert!(ack
9769            .subc_capabilities
9770            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9771        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9772        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9773        assert!(ack
9774            .subc_ops
9775            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9776        assert!(ack
9777            .subc_ops
9778            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9779
9780        let registration = registry.get_module("aft").unwrap().unwrap();
9781        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9782        assert_eq!(registration.state, ChannelState::Active);
9783        assert_eq!(registration.connection_id, conn);
9784        assert_eq!(registration.control_ops, module_baseline_control_ops());
9785    }
9786
9787    #[test]
9788    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9789        let invalid_identifiers = [
9790            ("case_change", "credentials-Provider/v1"),
9791            ("leading_zero", "credentials-provider/v01"),
9792            ("trailing_hyphen", "credentials-provider-/v1"),
9793            ("consecutive_hyphens", "credentials--provider/v1"),
9794            ("uppercase", "Credentials-provider/v1"),
9795            ("missing_v", "credentials-provider/1"),
9796            ("whitespace", "credentials provider/v1"),
9797            ("zero_version", "credentials-provider/v0"),
9798            ("out_of_range_version", "credentials-provider/v4294967296"),
9799            (
9800                "overlength_name",
9801                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9802            ),
9803        ];
9804        let mut cases = invalid_identifiers
9805            .into_iter()
9806            .map(|(name, identifier)| {
9807                (
9808                    format!("identifier_{name}"),
9809                    "capabilities.provides[0]".to_string(),
9810                    identifier.to_string(),
9811                    json!({ "provides": [identifier] }),
9812                    None,
9813                )
9814            })
9815            .collect::<Vec<_>>();
9816        cases.extend([
9817            (
9818                "unknown_need".to_string(),
9819                "capabilities.requires[0].need".to_string(),
9820                "deferred".to_string(),
9821                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9822                None,
9823            ),
9824            (
9825                "duplicate_provides".to_string(),
9826                "capabilities.provides[1]".to_string(),
9827                "credentials-provider/v1".to_string(),
9828                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9829                None,
9830            ),
9831            (
9832                "duplicate_must_never_reach".to_string(),
9833                "capabilities.must_never_reach[1]".to_string(),
9834                "credentials-provider/v1".to_string(),
9835                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9836                None,
9837            ),
9838            (
9839                "duplicate_requires_same_need".to_string(),
9840                "capabilities.requires[1]".to_string(),
9841                "credentials-provider/v1".to_string(),
9842                json!({ "requires": [
9843                    { "capability": "credentials-provider/v1", "need": "required" },
9844                    { "capability": "credentials-provider/v1", "need": "required" }
9845                ] }),
9846                None,
9847            ),
9848            (
9849                "duplicate_requires_conflicting_need".to_string(),
9850                "capabilities.requires[1]".to_string(),
9851                "credentials-provider/v1".to_string(),
9852                json!({ "requires": [
9853                    { "capability": "credentials-provider/v1", "need": "required" },
9854                    { "capability": "credentials-provider/v1", "need": "optional" }
9855                ] }),
9856                None,
9857            ),
9858            (
9859                "capabilities_root_pointer".to_string(),
9860                "runtime_computed[0]".to_string(),
9861                "/capabilities".to_string(),
9862                json!({}),
9863                Some(json!(["/capabilities"])),
9864            ),
9865            (
9866                "capabilities_descendant_pointer".to_string(),
9867                "runtime_computed[0]".to_string(),
9868                "/capabilities/provides".to_string(),
9869                json!({}),
9870                Some(json!(["/capabilities/provides"])),
9871            ),
9872            (
9873                "malformed_pointer_without_leading_slash".to_string(),
9874                "runtime_computed[0]".to_string(),
9875                "capabilities".to_string(),
9876                json!({}),
9877                Some(json!(["capabilities"])),
9878            ),
9879            (
9880                "malformed_pointer_escape".to_string(),
9881                "runtime_computed[0]".to_string(),
9882                "/roles/~2/tools".to_string(),
9883                json!({}),
9884                Some(json!(["/roles/~2/tools"])),
9885            ),
9886            (
9887                "unknown_capabilities_field".to_string(),
9888                "capabilities.future".to_string(),
9889                "<array>".to_string(),
9890                json!({ "future": [] }),
9891                None,
9892            ),
9893        ]);
9894
9895        for (index, (name, field, value, capabilities, runtime_computed)) in
9896            cases.into_iter().enumerate()
9897        {
9898            let registry = Arc::new(Registry::default());
9899            let handler = ControlHandler::new(Arc::clone(&registry));
9900            let response = handler
9901                .handle_control(
9902                    ConnectionId::new((index + 1) as u64),
9903                    capability_grammar_hello_frame(
9904                        capabilities,
9905                        runtime_computed,
9906                        index as u64 + 1,
9907                    ),
9908                )
9909                .expect("invalid HELLO returns a refusal");
9910
9911            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9912            let error = parse_error(&response[0]);
9913            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9914            let message = error["message"]
9915                .as_str()
9916                .expect("error message is a string");
9917            assert!(
9918                message.contains(&field),
9919                "{name}: field missing from {message}"
9920            );
9921            assert!(
9922                message.contains(&value),
9923                "{name}: value missing from {message}"
9924            );
9925            assert_eq!(
9926                registry
9927                    .active_registration_count()
9928                    .expect("registry reads"),
9929                0,
9930                "{name}: refused HELLO must not create a catalog entry"
9931            );
9932        }
9933    }
9934
9935    #[test]
9936    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9937        let registry = Arc::new(Registry::default());
9938        let handler = ControlHandler::new(Arc::clone(&registry));
9939        let capabilities = json!({
9940            "provides": ["credentials-provider/v1"],
9941            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9942            "must_never_reach": ["federation-transport/v1"]
9943        });
9944        let response = handler
9945            .handle_control(
9946                ConnectionId::new(99),
9947                capability_grammar_hello_frame(
9948                    capabilities.clone(),
9949                    Some(json!(["/roles/0/tools"])),
9950                    99,
9951                ),
9952            )
9953            .expect("valid HELLO registers");
9954        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9955
9956        let request = Frame::build(
9957            FrameType::Request,
9958            control_flags(),
9959            0,
9960            0,
9961            100,
9962            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9963                .expect("catalog request serializes"),
9964        )
9965        .expect("catalog request frame builds");
9966        let response = handler
9967            .handle_catalog_list(request, None)
9968            .expect("catalog list succeeds");
9969        let ClientControlResponse::CatalogList { modules, .. } =
9970            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9971        else {
9972            panic!("catalog request must return catalog.list");
9973        };
9974        assert_eq!(modules.len(), 1);
9975        assert_eq!(
9976            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9977            capabilities
9978        );
9979    }
9980
9981    #[test]
9982    fn catalog_list_mirrors_management_operation_description() {
9983        let registry = Arc::new(Registry::default());
9984        let handler = ControlHandler::new(Arc::clone(&registry));
9985        let description = "List managed records and return their identifiers and metadata.";
9986        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9987        manifest.provides = vec![ProviderRole::ManagementSurface {
9988            operations: vec![ManagementOperation {
9989                name: "records.list".to_string(),
9990                kind: ManagementOperationKind::Query,
9991                description: Some(description.to_string()),
9992            }],
9993            config_schema: json!({"type": "object"}),
9994            observability: vec![ObservabilitySurface {
9995                name: "records.stats".to_string(),
9996                kind: ObservabilityKind::Snapshot,
9997            }],
9998            identity_scope: vec![IdentityScope::Project],
9999            concurrency: Concurrency::ModuleManaged,
10000        }];
10001        registry
10002            .register_with_control_ops(
10003                manifest,
10004                PROTOCOL_VERSION,
10005                ConnectionId::new(99),
10006                Vec::new(),
10007            )
10008            .expect("described management manifest registers");
10009
10010        let request = Frame::build(
10011            FrameType::Request,
10012            control_flags(),
10013            0,
10014            0,
10015            100,
10016            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10017                .expect("catalog request serializes"),
10018        )
10019        .expect("catalog request frame builds");
10020        let response = handler
10021            .handle_catalog_list(request, None)
10022            .expect("catalog list succeeds");
10023        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10024        assert_eq!(
10025            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10026            "catalog.list must preserve the declared operation description verbatim"
10027        );
10028    }
10029
10030    #[test]
10031    fn catalog_list_mirrors_registered_operation_delivery() {
10032        let registry = Arc::new(Registry::default());
10033        let handler = ControlHandler::new(Arc::clone(&registry));
10034        let delivery =
10035            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10036        let mut body = serde_json::to_value(ModuleHelloBody {
10037            manifest: manifest("watch-management", PROTOCOL_VERSION),
10038            protocol_ver: PROTOCOL_VERSION,
10039            control_ops: None,
10040            launch_nonce: None,
10041        })
10042        .unwrap();
10043        body["manifest"]["provides"] = json!([{
10044            "role": "management_surface",
10045            "operations": [
10046                {"name": "status", "kind": "query"},
10047                {"name": "future", "kind": "query"},
10048                {"name": "reset", "kind": "mutate"}
10049            ],
10050            "config_schema": {}, "observability": [], "identity_scope": []
10051        }]);
10052        body["manifest"]["operation_delivery"] = delivery.clone();
10053        let hello = Frame::build(
10054            FrameType::Hello,
10055            control_flags(),
10056            0,
10057            0,
10058            99,
10059            serde_json::to_vec(&body).unwrap(),
10060        )
10061        .unwrap();
10062        let response = handler
10063            .handle_control(ConnectionId::new(99), hello)
10064            .unwrap();
10065        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10066
10067        let (_, registrations) = registry.list_modules().unwrap();
10068        let registered = &registrations[0].manifest;
10069        assert!(registered.is_watch_operation("status"));
10070        assert!(!registered.is_watch_operation("future"));
10071        assert!(!registered.is_watch_operation("reset"));
10072        assert!(!registered.is_watch_operation("missing"));
10073
10074        let request = Frame::build(
10075            FrameType::Request,
10076            control_flags(),
10077            0,
10078            0,
10079            100,
10080            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10081        )
10082        .unwrap();
10083        let response = handler.handle_catalog_list(request, None).unwrap();
10084        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10085        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10086        let ClientControlResponse::CatalogList { modules, .. } =
10087            serde_json::from_slice(&response[0].body).unwrap()
10088        else {
10089            panic!("expected catalog.list");
10090        };
10091        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10092    }
10093
10094    #[test]
10095    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10096        let registry = Arc::new(Registry::default());
10097        let supervisor = SupervisorHandle::new();
10098        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10099        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10100        let descriptor = json!({
10101            "schema": "ckios.richui.manifest/1",
10102            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10103            "bytes": 4096,
10104            "chunks": 4
10105        });
10106        let connection = ConnectionId::new(201);
10107        let hello = handler
10108            .handle_control(
10109                connection,
10110                hello_frame_with_rich_ui(
10111                    "vault",
10112                    PROTOCOL_VERSION,
10113                    201,
10114                    Some("the-real-nonce"),
10115                    descriptor.clone(),
10116                ),
10117            )
10118            .unwrap();
10119        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10120
10121        let request =
10122            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10123        let provides = registry
10124            .get_module_by_connection(connection)
10125            .unwrap()
10126            .unwrap()
10127            .manifest
10128            .provides;
10129        handler
10130            .handle_catalog_update(connection, request, provides, None, Some(false))
10131            .unwrap();
10132
10133        let request = Frame::build(
10134            FrameType::Request,
10135            control_flags(),
10136            0,
10137            0,
10138            203,
10139            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10140        )
10141        .unwrap();
10142        let response = handler.handle_catalog_list(request, None).unwrap();
10143        let ClientControlResponse::CatalogList { modules, .. } =
10144            serde_json::from_slice(&response[0].body).unwrap()
10145        else {
10146            panic!("expected catalog.list");
10147        };
10148        assert_eq!(
10149            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10150            descriptor
10151        );
10152        assert!(modules[0].nonce_verified);
10153    }
10154
10155    fn hello_with_role_declarations(declarations: Value) -> Frame {
10156        let mut body = serde_json::to_value(ModuleHelloBody {
10157            manifest: manifest("role-test", PROTOCOL_VERSION),
10158            protocol_ver: PROTOCOL_VERSION,
10159            control_ops: None,
10160            launch_nonce: None,
10161        })
10162        .unwrap();
10163        body["manifest"]["role_declarations"] = declarations;
10164        Frame::build(
10165            FrameType::Hello,
10166            control_flags(),
10167            0,
10168            0,
10169            301,
10170            serde_json::to_vec(&body).unwrap(),
10171        )
10172        .unwrap()
10173    }
10174
10175    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10176        let request = Frame::build(
10177            FrameType::Request,
10178            control_flags(),
10179            0,
10180            0,
10181            302,
10182            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10183        )
10184        .unwrap();
10185        let response = handler.handle_catalog_list(request, None).unwrap();
10186        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10187        let ClientControlResponse::CatalogList { mut modules, .. } =
10188            serde_json::from_slice(&response[0].body).unwrap()
10189        else {
10190            panic!("expected catalog.list");
10191        };
10192        (raw["modules"][0].clone(), modules.remove(0))
10193    }
10194
10195    #[test]
10196    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10197        let registry = Arc::new(Registry::default());
10198        let handler = ControlHandler::new(Arc::clone(&registry));
10199        let declarations = json!({
10200            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10201            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10202        });
10203        let connection = ConnectionId::new(301);
10204        let response = handler
10205            .handle_control(
10206                connection,
10207                hello_with_role_declarations(declarations.clone()),
10208            )
10209            .unwrap();
10210        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10211        let registered = registry
10212            .get_module_by_connection(connection)
10213            .unwrap()
10214            .unwrap()
10215            .manifest;
10216        // Compared as JSON values, not bytes: object key order is not part of
10217        // the relay's promise, and workspace builds unify serde_json's
10218        // preserve_order feature, which changes the order a literal keeps.
10219        assert_eq!(
10220            serde_json::to_value(&registered.role_declarations).unwrap(),
10221            declarations
10222        );
10223        let (raw, entry) = role_declarations_catalog(&handler);
10224        assert_eq!(raw["role_declarations"], declarations);
10225        assert_eq!(entry.role_declarations, registered.role_declarations);
10226
10227        let request =
10228            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10229        handler
10230            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10231            .unwrap();
10232        let (raw, entry) = role_declarations_catalog(&handler);
10233        assert_eq!(raw["role_declarations"], declarations);
10234        assert_eq!(entry.role_declarations, registered.role_declarations);
10235    }
10236
10237    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10238        let registry = Arc::new(Registry::default());
10239        let handler = ControlHandler::new(Arc::clone(&registry));
10240        let capture = EventCapture::default();
10241        let _subscriber =
10242            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10243        let response = handler
10244            .handle_control(
10245                ConnectionId::new(301),
10246                hello_with_role_declarations(declarations),
10247            )
10248            .unwrap();
10249        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10250        assert!(registry
10251            .get_module_by_connection(ConnectionId::new(301))
10252            .unwrap()
10253            .unwrap()
10254            .manifest
10255            .role_declarations
10256            .is_empty());
10257        let (raw, entry) = role_declarations_catalog(&handler);
10258        assert!(raw.get("role_declarations").is_none());
10259        assert!(entry.role_declarations.is_empty());
10260        let warnings: Vec<_> = capture
10261            .events()
10262            .into_iter()
10263            .filter(|event| {
10264                event.fields.get("message").map(String::as_str)
10265                    == Some("dropping whole role_declarations map from HELLO")
10266            })
10267            .collect();
10268        assert_eq!(warnings.len(), 1);
10269        assert_eq!(warnings[0].level, tracing::Level::WARN);
10270        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10271        assert_eq!(warnings[0].fields["reason"], reason);
10272    }
10273
10274    #[test]
10275    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10276        for value in [
10277            json!(null),
10278            json!(false),
10279            json!(0),
10280            json!("opaque"),
10281            json!([]),
10282        ] {
10283            assert_hello_drops_role_declarations_with_warning(
10284                json!({"routing/v1": {}, "broken/v1": value}),
10285                "role_declarations values must be objects",
10286            );
10287        }
10288        assert_hello_drops_role_declarations_with_warning(
10289            json!([]),
10290            "role_declarations must be an object",
10291        );
10292    }
10293
10294    #[test]
10295    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10296        assert_hello_drops_role_declarations_with_warning(
10297            json!({"routing/v1": {}, "bad key": {}}),
10298            "role_declarations key must use capability-name grammar",
10299        );
10300    }
10301
10302    #[test]
10303    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10304        assert_hello_drops_role_declarations_with_warning(
10305            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10306            "role_declarations exceeds 16384 compact JSON bytes",
10307        );
10308    }
10309
10310    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10311        let mut body = serde_json::to_value(ModuleHelloBody {
10312            manifest: manifest("flow-ops", PROTOCOL_VERSION),
10313            protocol_ver: PROTOCOL_VERSION,
10314            control_ops: None,
10315            launch_nonce: None,
10316        })
10317        .unwrap();
10318        body["manifest"]["provides"] = json!([{
10319            "role": "management_surface",
10320            "operations": [
10321                {"name": "diagnostics.findings", "kind": "query"},
10322                {"name": "reset", "kind": "mutate"}
10323            ],
10324            "config_schema": {}, "observability": [], "identity_scope": []
10325        }]);
10326        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10327        if let Some(flow_operations) = flow_operations {
10328            body["manifest"]["flow_operations"] = flow_operations;
10329        }
10330        Frame::build(
10331            FrameType::Hello,
10332            control_flags(),
10333            0,
10334            0,
10335            401,
10336            serde_json::to_vec(&body).unwrap(),
10337        )
10338        .unwrap()
10339    }
10340
10341    #[test]
10342    fn catalog_list_mirrors_flow_operations_unchanged() {
10343        // Order is kept as declared (not sorted) and an empty list stays an
10344        // affirmative "none", distinct from an absent declaration.
10345        for (capability, flow_operations) in [
10346            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10347            ("call-modes/v1", json!(["diagnostics.findings"])),
10348            ("flow-scopes/v1", json!([])),
10349        ] {
10350            let registry = Arc::new(Registry::default());
10351            let handler = ControlHandler::new(Arc::clone(&registry));
10352            let connection = ConnectionId::new(401);
10353            let response = handler
10354                .handle_control(
10355                    connection,
10356                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10357                )
10358                .unwrap();
10359            assert_eq!(response[0].header.ty, FrameType::HelloAck);
10360            let registered = registry
10361                .get_module_by_connection(connection)
10362                .unwrap()
10363                .unwrap()
10364                .manifest;
10365            assert_eq!(
10366                serde_json::to_value(&registered.flow_operations).unwrap(),
10367                flow_operations
10368            );
10369            let (raw, entry) = role_declarations_catalog(&handler);
10370            assert_eq!(raw["flow_operations"], flow_operations);
10371            assert_eq!(entry.flow_operations, registered.flow_operations);
10372        }
10373
10374        let registry = Arc::new(Registry::default());
10375        let handler = ControlHandler::new(Arc::clone(&registry));
10376        let response = handler
10377            .handle_control(
10378                ConnectionId::new(401),
10379                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10380            )
10381            .unwrap();
10382        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10383        let (raw, entry) = role_declarations_catalog(&handler);
10384        assert!(raw.get("flow_operations").is_none());
10385        assert_eq!(entry.flow_operations, None);
10386    }
10387
10388    fn assert_hello_drops_flow_operations_with_warning(
10389        capabilities: Value,
10390        flow_operations: Value,
10391        reason: &str,
10392    ) {
10393        let registry = Arc::new(Registry::default());
10394        let handler = ControlHandler::new(Arc::clone(&registry));
10395        let capture = EventCapture::default();
10396        let _subscriber =
10397            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10398        let response = handler
10399            .handle_control(
10400                ConnectionId::new(401),
10401                hello_with_flow_operations(capabilities, Some(flow_operations)),
10402            )
10403            .unwrap();
10404        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10405        let registered = registry
10406            .get_module_by_connection(ConnectionId::new(401))
10407            .unwrap()
10408            .expect("the module still registers")
10409            .manifest;
10410        assert_eq!(registered.module_id, "flow-ops");
10411        assert_eq!(registered.flow_operations, None);
10412        let (raw, entry) = role_declarations_catalog(&handler);
10413        assert_eq!(raw["module_id"], "flow-ops");
10414        assert!(raw.get("flow_operations").is_none());
10415        assert_eq!(entry.flow_operations, None);
10416        let warnings: Vec<_> = capture
10417            .events()
10418            .into_iter()
10419            .filter(|event| {
10420                event.fields.get("message").map(String::as_str)
10421                    == Some("dropping flow_operations from HELLO")
10422            })
10423            .collect();
10424        assert_eq!(warnings.len(), 1);
10425        assert_eq!(warnings[0].level, tracing::Level::WARN);
10426        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10427        assert_eq!(warnings[0].fields["reason"], reason);
10428    }
10429
10430    #[test]
10431    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10432        assert_hello_drops_flow_operations_with_warning(
10433            json!(["flow-scopes/v1"]),
10434            json!(["diagnostics.findings", "missing"]),
10435            "flow_operations names 'missing', which is not a declared management operation",
10436        );
10437    }
10438
10439    #[test]
10440    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10441        assert_hello_drops_flow_operations_with_warning(
10442            json!([]),
10443            json!(["diagnostics.findings"]),
10444            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10445        );
10446        // A run-only capability does not admit flows either.
10447        assert_hello_drops_flow_operations_with_warning(
10448            json!(["agent-run-scopes/v1"]),
10449            json!([]),
10450            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10451        );
10452    }
10453
10454    #[test]
10455    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10456        assert_hello_drops_flow_operations_with_warning(
10457            json!(["flow-scopes/v1"]),
10458            json!(["reset", "reset"]),
10459            "flow_operations names 'reset' more than once",
10460        );
10461        assert_hello_drops_flow_operations_with_warning(
10462            json!(["flow-scopes/v1"]),
10463            json!(vec!["reset"; 257]),
10464            "flow_operations lists 257 names, more than 256",
10465        );
10466        assert_hello_drops_flow_operations_with_warning(
10467            json!(["flow-scopes/v1"]),
10468            json!("reset"),
10469            "flow_operations must be an array of operation names",
10470        );
10471    }
10472
10473    #[test]
10474    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10475        let registry = Arc::new(Registry::default());
10476        let handler = ControlHandler::new(registry.clone());
10477        let descriptor = json!({
10478            "schema": "ckios.richui.manifest/1",
10479            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10480            "bytes": 128,
10481            "chunks": 2
10482        });
10483        let malformed = [
10484            ("schema", json!("")),
10485            ("schema", json!("a".repeat(65))),
10486            ("schema", json!("Bad schema")),
10487            (
10488                "manifest_digest",
10489                json!(format!("sha256:{}", "A".repeat(64))),
10490            ),
10491            ("manifest_digest", json!("sha256:abc")),
10492            ("bytes", json!(0)),
10493            ("bytes", json!(16_777_217)),
10494            ("bytes", json!("128")),
10495            ("chunks", json!(0)),
10496            ("chunks", json!(1025)),
10497            ("chunks", json!("2")),
10498        ];
10499        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10500            let mut invalid = descriptor.clone();
10501            invalid[field] = invalid_value;
10502            let response = handler
10503                .handle_control(
10504                    ConnectionId::new(210 + index as u64),
10505                    hello_frame_with_rich_ui(
10506                        &format!("bad-rich-ui-{index}"),
10507                        PROTOCOL_VERSION,
10508                        210 + index as u64,
10509                        None,
10510                        invalid,
10511                    ),
10512                )
10513                .unwrap();
10514            assert_eq!(response[0].header.ty, FrameType::Error);
10515            let error = parse_error(&response[0]);
10516            assert_eq!(error["code"], "invalid_manifest");
10517            assert!(error["message"]
10518                .as_str()
10519                .unwrap()
10520                .contains(&format!("rich_ui.{field}")));
10521            assert!(registry
10522                .get_module(&format!("bad-rich-ui-{index}"))
10523                .unwrap()
10524                .is_none());
10525        }
10526    }
10527
10528    #[test]
10529    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10530        let registry = Arc::new(Registry::default());
10531        let handler = ControlHandler::new(Arc::clone(&registry));
10532        let connection = ConnectionId::new(219);
10533        registry
10534            .register_with_control_ops(
10535                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10536                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10537                        "bad schema",
10538                        format!("sha256:{}", "a".repeat(64)),
10539                        128,
10540                        2,
10541                    )))
10542                    .build(),
10543                PROTOCOL_VERSION,
10544                connection,
10545                Vec::new(),
10546            )
10547            .unwrap();
10548        let request =
10549            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10550        let response = handler
10551            .handle_catalog_update(connection, request, Vec::new(), None, None)
10552            .unwrap();
10553        assert_eq!(response[0].header.ty, FrameType::Error);
10554        let error = parse_error(&response[0]);
10555        assert_eq!(error["code"], "invalid_manifest");
10556        assert!(error["message"]
10557            .as_str()
10558            .unwrap()
10559            .contains("rich_ui.schema"));
10560    }
10561
10562    #[test]
10563    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10564        let registry = Arc::new(Registry::default());
10565        let supervisor = SupervisorHandle::new();
10566        supervisor.open_swap("provider", "candidate-nonce".to_string());
10567        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10568        let connection = ConnectionId::new(218);
10569        let hello = handler
10570            .handle_control(
10571                connection,
10572                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10573            )
10574            .unwrap();
10575        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10576        assert!(registry.list_modules().unwrap().1.is_empty());
10577
10578        registry.promote_candidate("provider").unwrap().unwrap();
10579        let request = Frame::build(
10580            FrameType::Request,
10581            control_flags(),
10582            0,
10583            0,
10584            219,
10585            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10586        )
10587        .unwrap();
10588        let response = handler.handle_catalog_list(request, None).unwrap();
10589        let ClientControlResponse::CatalogList { modules, .. } =
10590            serde_json::from_slice(&response[0].body).unwrap()
10591        else {
10592            panic!("expected catalog.list");
10593        };
10594        assert_eq!(modules.len(), 1);
10595        assert!(modules[0].nonce_verified);
10596    }
10597
10598    #[test]
10599    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10600        for provenance_json in [
10601            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10602            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10603        ] {
10604            let registry = Arc::new(Registry::default());
10605            let handler = ControlHandler::new(registry);
10606            let hello = Frame::build(
10607                FrameType::Hello,
10608                control_flags(),
10609                0,
10610                0,
10611                220,
10612                serde_json::to_vec(&ModuleHelloBody {
10613                    manifest: ModuleManifest::builder("built", "0.1.0")
10614                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10615                        .build(),
10616                    protocol_ver: PROTOCOL_VERSION,
10617                    control_ops: None,
10618                    launch_nonce: None,
10619                })
10620                .unwrap(),
10621            )
10622            .unwrap();
10623            assert_eq!(
10624                handler
10625                    .handle_control(ConnectionId::new(220), hello)
10626                    .unwrap()[0]
10627                    .header
10628                    .ty,
10629                FrameType::HelloAck
10630            );
10631            let request = Frame::build(
10632                FrameType::Request,
10633                control_flags(),
10634                0,
10635                0,
10636                221,
10637                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10638            )
10639            .unwrap();
10640            let response = handler.handle_catalog_list(request, None).unwrap();
10641            let ClientControlResponse::CatalogList { modules, .. } =
10642                serde_json::from_slice(&response[0].body).unwrap()
10643            else {
10644                panic!("expected catalog.list");
10645            };
10646            assert_eq!(modules.len(), 1);
10647            assert_eq!(
10648                serde_json::to_vec(&modules[0]).unwrap(),
10649                format!(
10650                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10651                )
10652                .as_bytes()
10653            );
10654        }
10655    }
10656
10657    #[test]
10658    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10659        let registry = Arc::new(Registry::default());
10660        let handler = ControlHandler::new(Arc::clone(&registry));
10661        let hello = Frame::build(
10662            FrameType::Hello,
10663            control_flags(),
10664            0,
10665            0,
10666            220,
10667            serde_json::to_vec(&ModuleHelloBody {
10668                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10669                protocol_ver: PROTOCOL_VERSION,
10670                control_ops: None,
10671                launch_nonce: Some("unverified".to_string()),
10672            })
10673            .unwrap(),
10674        )
10675        .unwrap();
10676        assert_eq!(
10677            handler
10678                .handle_control(ConnectionId::new(220), hello)
10679                .unwrap()[0]
10680                .header
10681                .ty,
10682            FrameType::HelloAck
10683        );
10684        let request = Frame::build(
10685            FrameType::Request,
10686            control_flags(),
10687            0,
10688            0,
10689            221,
10690            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10691        )
10692        .unwrap();
10693        let response = handler.handle_catalog_list(request, None).unwrap();
10694        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10695        assert!(wire["modules"][0].get("provenance").is_none());
10696        let ClientControlResponse::CatalogList { modules, .. } =
10697            serde_json::from_slice(&response[0].body).unwrap()
10698        else {
10699            panic!("expected catalog.list");
10700        };
10701        assert_eq!(
10702            serde_json::to_vec(&modules[0]).unwrap(),
10703            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10704        );
10705        assert!(!modules[0].nonce_verified);
10706        assert!(modules[0].provenance.is_none());
10707    }
10708
10709    #[test]
10710    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10711        let registry = Arc::new(Registry::default());
10712        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10713            [("vault".to_string(), true), ("squatter".to_string(), true)],
10714            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10715        );
10716        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10717        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10718            provides: vec!["credentials-provider/v1".to_string()],
10719            requires: Vec::new(),
10720            must_never_reach: Vec::new(),
10721        });
10722        let frame = Frame::build(
10723            FrameType::Hello,
10724            control_flags(),
10725            0,
10726            0,
10727            77,
10728            serde_json::to_vec(&ModuleHelloBody {
10729                manifest: squatter,
10730                protocol_ver: PROTOCOL_VERSION,
10731                control_ops: None,
10732                launch_nonce: None,
10733            })
10734            .expect("HELLO serializes"),
10735        )
10736        .expect("HELLO frame builds");
10737        let response = handler
10738            .handle_control(ConnectionId::new(77), frame)
10739            .expect("reserved claim receives a typed refusal");
10740        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10741        assert_eq!(
10742            registry
10743                .active_registration_count()
10744                .expect("registry reads"),
10745            0,
10746            "a reserved capability refusal must not leave a catalog entry"
10747        );
10748    }
10749
10750    #[test]
10751    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10752        for settlement in ["timeout", "inconclusive", "drop"] {
10753            let breakers = RouteBindBreakers::default();
10754            let RouteBindAdmission::Admitted {
10755                guard: mut old,
10756                probe: false,
10757            } = breakers.admit("prov")
10758            else {
10759                panic!("ordinary relay admitted")
10760            };
10761            let RouteBindAdmission::Admitted {
10762                guard: mut opener, ..
10763            } = breakers.admit("prov")
10764            else {
10765                panic!("second relay admitted")
10766            };
10767            assert!(
10768                !opener
10769                    .record_timeout(1, Duration::ZERO)
10770                    .unwrap()
10771                    .reopened_after_probe
10772            );
10773            let RouteBindAdmission::Admitted {
10774                guard: mut probe,
10775                probe: true,
10776            } = breakers.admit("prov")
10777            else {
10778                panic!("one cooldown probe admitted")
10779            };
10780            match settlement {
10781                "timeout" => assert!(
10782                    !old.record_timeout(1, Duration::ZERO)
10783                        .unwrap()
10784                        .reopened_after_probe
10785                ),
10786                "inconclusive" => old.record_inconclusive(),
10787                "drop" => drop(old),
10788                _ => unreachable!(),
10789            }
10790            assert!(
10791                matches!(
10792                    breakers.admit("prov"),
10793                    RouteBindAdmission::Refused {
10794                        probe_in_flight: true,
10795                        ..
10796                    }
10797                ),
10798                "{settlement} of a pre-open relay cannot release the real probe"
10799            );
10800            assert!(
10801                probe
10802                    .record_timeout(1, Duration::ZERO)
10803                    .unwrap()
10804                    .reopened_after_probe
10805            );
10806            assert!(matches!(
10807                breakers.admit("prov"),
10808                RouteBindAdmission::Admitted { probe: true, .. }
10809            ));
10810        }
10811        let breakers = RouteBindBreakers::default();
10812        let admit = || match breakers.admit("prov") {
10813            RouteBindAdmission::Admitted { guard, .. } => guard,
10814            _ => panic!("relay admitted"),
10815        };
10816        admit().record_timeout(1, Duration::ZERO);
10817        let mut old_probe = admit();
10818        breakers.reset_for_new_module_connection("prov");
10819        admit().record_timeout(1, Duration::ZERO);
10820        let _new_probe = admit();
10821        old_probe.record_inconclusive();
10822        assert!(matches!(
10823            breakers.admit("prov"),
10824            RouteBindAdmission::Refused {
10825                probe_in_flight: true,
10826                ..
10827            }
10828        ));
10829    }
10830
10831    #[tokio::test]
10832    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10833        for candidate in [false, true] {
10834            let registry = Arc::new(Registry::default());
10835            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10836                [("vault".to_string(), true), ("squatter".to_string(), true)],
10837                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10838            );
10839            let conn = ConnectionId::new(77);
10840            let (ctx, mut rx) = route_ctx(conn);
10841            let initial = capability_manifest("squatter", &[], &[]);
10842            if candidate {
10843                registry
10844                    .register_candidate_with_control_ops(
10845                        initial.clone(),
10846                        PROTOCOL_VERSION,
10847                        conn,
10848                        module_baseline_control_ops(),
10849                    )
10850                    .unwrap();
10851                handler
10852                    .forwarding
10853                    .register_candidate_module_connection(
10854                        conn,
10855                        "squatter".to_string(),
10856                        PROTOCOL_VERSION,
10857                        manifest_concurrency(&initial),
10858                        ctx.egress.clone(),
10859                    )
10860                    .unwrap();
10861            } else {
10862                hello_via_sink(
10863                    &handler,
10864                    &ctx,
10865                    &mut rx,
10866                    hello_frame_with_manifest(initial.clone(), 1),
10867                )
10868                .await;
10869            }
10870            let response = handler
10871                .handle_control_frame(
10872                    &ctx,
10873                    catalog_update_with_capabilities_frame(
10874                        2,
10875                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
10876                            .capabilities
10877                            .unwrap(),
10878                    ),
10879                )
10880                .await
10881                .unwrap();
10882            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10883            assert_eq!(
10884                registry
10885                    .get_module_by_connection(conn)
10886                    .unwrap()
10887                    .unwrap()
10888                    .manifest,
10889                initial
10890            );
10891        }
10892    }
10893
10894    #[test]
10895    fn server_describe_surfaces_required_capability_verdict_fields() {
10896        let registry = Arc::new(Registry::default());
10897        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10898            [
10899                ("consumer".to_string(), true),
10900                ("provider".to_string(), false),
10901            ],
10902            BTreeMap::new(),
10903        );
10904        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10905        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10906            provides: Vec::new(),
10907            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10908                capability: "credentials-provider/v1".to_string(),
10909                need: subc_protocol::manifest::CapabilityNeed::Required,
10910            }],
10911            must_never_reach: Vec::new(),
10912        });
10913        let hello = Frame::build(
10914            FrameType::Hello,
10915            control_flags(),
10916            0,
10917            0,
10918            78,
10919            serde_json::to_vec(&ModuleHelloBody {
10920                manifest: consumer,
10921                protocol_ver: PROTOCOL_VERSION,
10922                control_ops: None,
10923                launch_nonce: None,
10924            })
10925            .expect("HELLO serializes"),
10926        )
10927        .expect("HELLO frame builds");
10928        handler
10929            .handle_control(ConnectionId::new(78), hello)
10930            .expect("consumer registers");
10931        let describe = Frame::build(
10932            FrameType::Request,
10933            control_flags(),
10934            0,
10935            0,
10936            79,
10937            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10938                .expect("request serializes"),
10939        )
10940        .expect("describe frame builds");
10941        let response = handler
10942            .handle_server_describe(describe)
10943            .expect("server.describe succeeds");
10944        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10945        let requirement = &rendered["capability_requirements"][0];
10946        assert_eq!(requirement["consumer"], "consumer");
10947        assert_eq!(requirement["verdict"], "never_provided");
10948        assert_eq!(requirement["episode_seq"], 1);
10949        assert_eq!(requirement["config_satisfiable"], false);
10950        assert_eq!(requirement["runtime_available"], false);
10951        assert!(requirement["detail"]
10952            .as_str()
10953            .expect("detail string")
10954            .contains("credentials-provider/v1"));
10955    }
10956
10957    /// Register a consumer that requires `project-identity/v1` and the provider
10958    /// of it, optionally mark daemon shutdown as begun, then deregister the
10959    /// provider as its connection closes. Returns the `capability.requirement`
10960    /// events that deregistration logged and the verdict the evaluator then
10961    /// holds for the requirement (the one `route.open` refusals read).
10962    fn deregister_required_provider(
10963        daemon_shutdown_begun: bool,
10964    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10965        const CAPABILITY: &str = "project-identity/v1";
10966        let registry = Arc::new(Registry::default());
10967        let daemon_shutdown = DaemonShutdownFlag::default();
10968        let handler = ControlHandler::new(Arc::clone(&registry))
10969            .with_capability_config(
10970                [
10971                    ("consumer".to_string(), true),
10972                    ("provider".to_string(), true),
10973                ],
10974                BTreeMap::new(),
10975            )
10976            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10977        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10978        consumer.capabilities = Some(CapabilityDeclarations {
10979            provides: Vec::new(),
10980            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10981                capability: CAPABILITY.to_string(),
10982                need: subc_protocol::manifest::CapabilityNeed::Required,
10983            }],
10984            must_never_reach: Vec::new(),
10985        });
10986        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10987        let provider_conn = ConnectionId::new(182);
10988        for (conn, manifest, corr) in [
10989            (ConnectionId::new(181), consumer, 181),
10990            (provider_conn, provider, 182),
10991        ] {
10992            let hello = handler
10993                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10994                .expect("module registers");
10995            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10996        }
10997        assert_eq!(
10998            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10999            Some(CapabilityVerdict::Provided),
11000            "the registered provider must satisfy the requirement first"
11001        );
11002        if daemon_shutdown_begun {
11003            daemon_shutdown.set_for_test();
11004        }
11005
11006        let capture = EventCapture::default();
11007        let _subscriber =
11008            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11009        handler
11010            .cleanup_connection(provider_conn)
11011            .expect("provider deregisters");
11012        let events = capture
11013            .events()
11014            .into_iter()
11015            .filter(|event| {
11016                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11017            })
11018            .collect();
11019        (
11020            events,
11021            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11022        )
11023    }
11024
11025    #[test]
11026    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11027        let (events, verdict) = deregister_required_provider(true);
11028        assert_eq!(
11029            verdict,
11030            Some(CapabilityVerdict::NeverProvided),
11031            "shutdown must not change the verdict route.open refusals read"
11032        );
11033        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11034        let event = &events[0];
11035        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11036        assert_eq!(
11037            event.fields.get("cause").map(String::as_str),
11038            Some("\"daemon_shutdown\""),
11039            "{event:?}"
11040        );
11041        assert_eq!(
11042            event.fields.get("verdict").map(String::as_str),
11043            Some("\"never_provided\""),
11044            "{event:?}"
11045        );
11046    }
11047
11048    #[test]
11049    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11050        let (events, verdict) = deregister_required_provider(false);
11051        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11052        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11053        let event = &events[0];
11054        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11055        assert_eq!(event.fields.get("cause"), None, "{event:?}");
11056        assert_eq!(
11057            event.fields.get("verdict").map(String::as_str),
11058            Some("\"never_provided\""),
11059            "{event:?}"
11060        );
11061    }
11062
11063    #[test]
11064    fn catalog_list_omits_capabilities_for_legacy_manifest() {
11065        let registry = Arc::new(Registry::default());
11066        let handler = ControlHandler::new(Arc::clone(&registry));
11067        let hello = handler
11068            .handle_control(
11069                ConnectionId::new(101),
11070                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11071            )
11072            .expect("legacy HELLO registers");
11073        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11074
11075        let request = Frame::build(
11076            FrameType::Request,
11077            control_flags(),
11078            0,
11079            0,
11080            102,
11081            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11082                .expect("catalog request serializes"),
11083        )
11084        .expect("catalog request frame builds");
11085        let response = handler
11086            .handle_catalog_list(request, None)
11087            .expect("catalog list succeeds");
11088        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11089        assert!(
11090            body["modules"][0].get("capabilities").is_none(),
11091            "legacy manifest must retain an absent capabilities field on catalog.list"
11092        );
11093    }
11094
11095    #[test]
11096    fn hello_ack_omits_storage_when_no_storage_config() {
11097        let registry = Arc::new(Registry::default());
11098        let handler = ControlHandler::new(Arc::clone(&registry));
11099        let responses = handler
11100            .handle_control(
11101                ConnectionId::new(1),
11102                hello_frame("aft", PROTOCOL_VERSION, 7),
11103            )
11104            .unwrap();
11105        let ack = parse_ack(&responses[0]);
11106        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11107        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11108    }
11109
11110    #[tokio::test]
11111    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11112        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11113        let registry = Arc::new(Registry::default());
11114        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
11115        let responses = handler
11116            .handle_control(
11117                ConnectionId::new(1),
11118                hello_frame("aft", PROTOCOL_VERSION, 7),
11119            )
11120            .unwrap();
11121        let ack = parse_ack(&responses[0]);
11122        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11123
11124        let described = handler
11125            .handle_control_frame(
11126                &route_ctx(ConnectionId::new(2)).0,
11127                Frame::build(
11128                    FrameType::Request,
11129                    control_flags(),
11130                    0,
11131                    0,
11132                    9,
11133                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11134                )
11135                .unwrap(),
11136            )
11137            .await
11138            .unwrap();
11139        let ClientControlResponse::ServerDescribe { machine_id, .. } =
11140            serde_json::from_slice(&described[0].body).unwrap()
11141        else {
11142            panic!("server.describe answered with another shape");
11143        };
11144        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11145    }
11146
11147    #[test]
11148    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11149        // With a central sqlite storage policy, each registering module gets its
11150        // own resolved descriptor in HELLO_ACK, keyed by its module id.
11151        let registry = Arc::new(Registry::default());
11152        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
11153            crate::daemon_config::StorageConfig::Sqlite {
11154                data_home: std::path::PathBuf::from("/data"),
11155            },
11156        ));
11157
11158        let responses = handler
11159            .handle_control(
11160                ConnectionId::new(1),
11161                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11162            )
11163            .unwrap();
11164        let ack = parse_ack(&responses[0]);
11165        assert_eq!(
11166            ack.storage,
11167            Some(serde_json::json!({
11168                "module_id": "alfonso-routing",
11169                "storage_namespace": "default",
11170                "isolation": { "kind": "module" },
11171                "backend": {
11172                    "backend": "sqlite",
11173                    "path": "/data/cortexkit/alfonso-routing/store.db"
11174                }
11175            })),
11176            "the delivered descriptor is the module's own sqlite store path"
11177        );
11178    }
11179
11180    #[test]
11181    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11182        let registry = Arc::new(Registry::default());
11183        let handler = ControlHandler::new(Arc::clone(&registry));
11184        let conn = ConnectionId::new(1);
11185        let responses = handler
11186            .handle_control(
11187                conn,
11188                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11189            )
11190            .unwrap();
11191        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11192        let registration = registry.get_module("aft").unwrap().unwrap();
11193        assert_eq!(registration.control_ops, module_baseline_control_ops());
11194
11195        let frame =
11196            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11197        assert!(handler
11198            .guard_module_control_op(&frame, "aft", "route.bind")
11199            .unwrap()
11200            .is_none());
11201        let error = handler
11202            .guard_module_control_op(&frame, "aft", "test.synthetic")
11203            .unwrap()
11204            .expect("synthetic ungranted op should be rejected");
11205        assert_eq!(error.header.ty, FrameType::Error);
11206        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11207    }
11208
11209    #[test]
11210    fn hello_control_ops_some_adds_optional_grants() {
11211        let registry = Arc::new(Registry::default());
11212        let handler = ControlHandler::new(Arc::clone(&registry));
11213        handler
11214            .handle_control(
11215                ConnectionId::new(1),
11216                hello_frame_with_control_ops(
11217                    "aft",
11218                    PROTOCOL_VERSION,
11219                    7,
11220                    Some(vec![
11221                        "future.synthetic".to_string(),
11222                        "route.bind".to_string(),
11223                    ]),
11224                ),
11225            )
11226            .unwrap();
11227        let registration = registry.get_module("aft").unwrap().unwrap();
11228        assert_eq!(
11229            registration.control_ops,
11230            vec![
11231                "route.bind".to_string(),
11232                "route.status".to_string(),
11233                "future.synthetic".to_string(),
11234            ]
11235        );
11236        let frame =
11237            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11238        assert!(handler
11239            .guard_module_control_op(&frame, "aft", "future.synthetic")
11240            .unwrap()
11241            .is_none());
11242    }
11243
11244    #[tokio::test]
11245    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11246        let registry = Arc::new(Registry::default());
11247        let forwarding = Arc::new(ForwardingTable::default());
11248        let handler =
11249            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11250        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11251        hello_via_sink(
11252            &handler,
11253            &module_ctx,
11254            &mut module_rx,
11255            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11256        )
11257        .await;
11258
11259        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11260        let responses = handler
11261            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11262            .await
11263            .unwrap();
11264        assert_eq!(responses.len(), 1);
11265        assert_eq!(responses[0].header.ty, FrameType::Error);
11266        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11267        assert!(module_rx.try_recv().is_err());
11268    }
11269
11270    #[tokio::test]
11271    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11272        let registry = Arc::new(Registry::default());
11273        let forwarding = Arc::new(ForwardingTable::default());
11274        let handler =
11275            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11276        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11277        hello_via_sink(
11278            &handler,
11279            &module_ctx,
11280            &mut module_rx,
11281            hello_frame_with_control_ops(
11282                "aft",
11283                PROTOCOL_VERSION,
11284                7,
11285                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11286            ),
11287        )
11288        .await;
11289
11290        let project_root = unique_project_root("demux");
11291        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11292        let route_handler = handler.clone();
11293        let route_task = tokio::spawn(async move {
11294            route_handler
11295                .handle_control_frame(
11296                    &route_client_ctx,
11297                    route_open_frame(100, "aft", project_root),
11298                )
11299                .await
11300                .unwrap()
11301        });
11302        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11303            .await
11304            .unwrap()
11305            .unwrap();
11306        assert!(matches!(
11307            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11308            ModuleControlRequest::RouteBind { .. }
11309        ));
11310
11311        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11312        let health_handler = handler.clone();
11313        let health_task = tokio::spawn(async move {
11314            health_handler
11315                .handle_control_frame(
11316                    &health_client_ctx,
11317                    supervisor_health_probe_frame(101, "aft"),
11318                )
11319                .await
11320                .unwrap()
11321        });
11322        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11323            .await
11324            .unwrap()
11325            .unwrap();
11326        assert_eq!(
11327            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11328            ModuleControlRequest::HealthCheck {}
11329        );
11330
11331        handler
11332            .handle_control_frame(
11333                &module_ctx,
11334                health_response(health_frame.header.corr, HealthStatus::Degraded),
11335            )
11336            .await
11337            .unwrap();
11338        let health_response = health_task.await.unwrap();
11339        assert_eq!(health_response.len(), 1);
11340        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11341            ClientControlResponse::SupervisorHealthProbe {
11342                module_id,
11343                status,
11344                detail,
11345                metrics,
11346            } => {
11347                assert_eq!(module_id, "aft");
11348                assert_eq!(status, HealthStatus::Degraded);
11349                assert_eq!(detail.as_deref(), Some("warming"));
11350                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11351            }
11352            other => panic!("unexpected health response: {other:?}"),
11353        }
11354
11355        handler
11356            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11357            .await
11358            .unwrap();
11359        let route_response = route_task.await.unwrap();
11360        assert!(route_response.is_empty());
11361        let published = route_client_rx.recv().await.unwrap();
11362        assert!(matches!(
11363            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11364            ClientControlResponse::RouteOpen { .. }
11365        ));
11366    }
11367
11368    /// Start one `route.open` on `client_connection` and return its still-running
11369    /// handler task together with the `route.bind` the module received for it.
11370    /// The handler blocks until the module answers, so it has to run as a task
11371    /// while the test drives the module side.
11372    async fn relay_route_open(
11373        handler: &ControlHandler,
11374        client_connection: ConnectionId,
11375        client_egress: &FrameSink,
11376        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11377        corr: u64,
11378        module_id: &str,
11379        project_root_label: &str,
11380    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11381        let ctx = RouteCtx {
11382            connection_id: client_connection,
11383            egress: client_egress.clone(),
11384        };
11385        let handler = handler.clone();
11386        let project_root = unique_project_root(project_root_label);
11387        let module_id = module_id.to_string();
11388        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11389        let task = tokio::spawn(async move {
11390            let _guard = tracing::dispatcher::set_default(&dispatch);
11391            handler
11392                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11393                .await
11394                .unwrap()
11395        });
11396        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11397            .await
11398            .expect("module receives the relayed route.bind")
11399            .expect("module egress is open");
11400        (task, bind.frame)
11401    }
11402
11403    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11404        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11405            ModuleControlRequest::RouteBind {
11406                route_channel,
11407                epoch,
11408                ..
11409            } => (route_channel, epoch),
11410            other => panic!("expected a route.bind request, got {other:?}"),
11411        }
11412    }
11413
11414    fn published_route(frame: &Frame) -> (u16, u32) {
11415        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11416            ClientControlResponse::RouteOpen {
11417                route_channel,
11418                route_epoch,
11419                ..
11420            } => (route_channel, route_epoch),
11421            other => panic!("expected a route.open response, got {other:?}"),
11422        }
11423    }
11424
11425    /// Reproduction of a production outage. A client had `route.open`s in
11426    /// flight to a module and was already marked closing -- its egress had refused a
11427    /// module frame, so the daemon asked its connection to end -- while its sink
11428    /// was still open. When the module acked those binds, the daemon refused to
11429    /// commit a route for a closing client, and that refusal was returned from
11430    /// the MODULE connection's frame handler, where a router error that has no
11431    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
11432    /// the supervisor correctly did not respawn a clean exit, and every seat lost
11433    /// its tools for hours -- one client's teardown took down a connection
11434    /// carrying ~170 other routes.
11435    ///
11436    /// The window is opened here by calling the production path that opens it
11437    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
11438    /// state that matters is "in `closing_connections`, sink still open, relay
11439    /// still pending", and it lasts only from the close request until the
11440    /// connection loop reacts to it; a socket-level test can flood a client into
11441    /// that escalation but cannot pin the module's ack inside the window. Closing
11442    /// the socket instead takes the other path entirely -- connection teardown
11443    /// removes the pending relay under the same lock, so the ack finds nothing.
11444    #[tokio::test]
11445    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11446        let registry = Arc::new(Registry::default());
11447        let forwarding = Arc::new(ForwardingTable::default());
11448        let handler =
11449            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11450
11451        let module_connection = ConnectionId::new(30);
11452        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11453        hello_via_sink(
11454            &handler,
11455            &module_ctx,
11456            &mut module_rx,
11457            hello_frame("aft", PROTOCOL_VERSION, 7),
11458        )
11459        .await;
11460
11461        let dying_client = ConnectionId::new(31);
11462        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11463
11464        // A published route on the dying client. The escalation below only marks
11465        // a connection closing for a route it has already published.
11466        let (first_task, first_bind) = relay_route_open(
11467            &handler,
11468            dying_client,
11469            &dying_ctx.egress,
11470            &mut module_rx,
11471            100,
11472            "aft",
11473            "closing-first",
11474        )
11475        .await;
11476        handler
11477            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11478            .await
11479            .unwrap();
11480        assert!(first_task.await.unwrap().is_empty());
11481        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11482
11483        // A second route.open from the same client, relayed and awaiting its ack.
11484        let (second_task, second_bind) = relay_route_open(
11485            &handler,
11486            dying_client,
11487            &dying_ctx.egress,
11488            &mut module_rx,
11489            101,
11490            "aft",
11491            "closing-second",
11492        )
11493        .await;
11494        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11495
11496        // The window: the client is closing, its sink is still open, and its
11497        // second bind is still pending.
11498        assert!(forwarding
11499            .escalate_client_delivery_failure(
11500                dying_client,
11501                first_channel,
11502                first_epoch,
11503                CloseReason::new(
11504                    "module_to_client_delivery_failed",
11505                    "client egress refused a module frame",
11506                ),
11507                crate::forwarding::UndeliveredFrame {
11508                    module_id: None,
11509                    sink: &dying_ctx.egress,
11510                },
11511            )
11512            .unwrap());
11513        assert!(!dying_ctx.egress.is_closed());
11514
11515        // The frame that used to end the module connection.
11516        let ack = handler
11517            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11518            .await;
11519        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11520        if module_loop_error.is_some() {
11521            // What the server's connection loop does with a router error that has
11522            // no ERROR-frame translation: end the connection, which releases the
11523            // module's registration and every route on it.
11524            handler.cleanup_connection(module_connection).unwrap();
11525        }
11526        // Read the module's next frame before opening the co-tenant's route, so
11527        // the GOODBYE assertion below is about THIS ack and not about later
11528        // traffic. `None` means the module was told nothing.
11529        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11530            .await
11531            .ok()
11532            .flatten();
11533
11534        // 1. The module connection is still registered.
11535        assert!(
11536            registry
11537                .get_module_by_connection(module_connection)
11538                .unwrap()
11539                .is_some(),
11540            "one client's closing connection ended the shared module connection: \
11541             {module_loop_error:?}"
11542        );
11543        // ...and still serving: another client can open and use a route on it.
11544        let cotenant = ConnectionId::new(32);
11545        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11546        let (cotenant_task, cotenant_bind) = relay_route_open(
11547            &handler,
11548            cotenant,
11549            &cotenant_ctx.egress,
11550            &mut module_rx,
11551            102,
11552            "aft",
11553            "closing-cotenant",
11554        )
11555        .await;
11556        handler
11557            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11558            .await
11559            .unwrap();
11560        assert!(cotenant_task.await.unwrap().is_empty());
11561        let (cotenant_channel, cotenant_epoch) =
11562            published_route(&cotenant_rx.recv().await.unwrap());
11563        assert!(matches!(
11564            forwarding
11565                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11566                .unwrap(),
11567            DataRoute::Client(DataRouteState::Bound(_))
11568        ));
11569
11570        // 2. The module was told to drop the binding it created for the route
11571        //    that will never be published.
11572        let goodbye = post_ack_module_frame
11573            .expect("module receives a GOODBYE for the abandoned route channel");
11574        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11575        assert_eq!(goodbye.header.channel, abandoned_channel);
11576        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11577
11578        // 3. The dying client received nothing: no route was ever published to
11579        //    it. Its route.open is answered as unavailable, which the connection
11580        //    loop would write to a socket that is already going away.
11581        assert!(dying_rx.try_recv().is_err());
11582        let second_response = second_task.await.unwrap();
11583        assert_eq!(second_response.len(), 1);
11584        assert_eq!(
11585            parse_error(&second_response[0])["code"],
11586            "target_unavailable"
11587        );
11588    }
11589
11590    /// One `catalog.list` as a client sees it: the generation and the listed
11591    /// module ids with their readiness.
11592    async fn catalog_snapshot(
11593        handler: &ControlHandler,
11594        client_ctx: &RouteCtx,
11595        corr: u64,
11596    ) -> (u64, Vec<(String, bool)>) {
11597        let body =
11598            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11599        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11600        let reply = handler
11601            .handle_control_frame(client_ctx, request)
11602            .await
11603            .unwrap()
11604            .pop()
11605            .unwrap();
11606        let ClientControlResponse::CatalogList {
11607            generation,
11608            modules,
11609            ..
11610        } = serde_json::from_slice(&reply.body).unwrap()
11611        else {
11612            panic!("not a catalog.list reply");
11613        };
11614        (
11615            generation,
11616            modules
11617                .into_iter()
11618                .map(|entry| (entry.module_id, entry.ready))
11619                .collect(),
11620        )
11621    }
11622
11623    /// A handler whose HELLO for module "aft" on connection 40 is parked
11624    /// between reserving its registry entry and installing its forwarding
11625    /// endpoint. Returns the handler, the two-party barrier (wait once to know
11626    /// the HELLO is parked, once more to release it), the HELLO's thread, and
11627    /// the module's context and receiver.
11628    fn hello_held_before_forwarding() -> (
11629        ControlHandler,
11630        Arc<std::sync::Barrier>,
11631        std::thread::JoinHandle<Vec<Frame>>,
11632        RouteCtx,
11633        mpsc::Receiver<crate::router::OutboundFrame>,
11634    ) {
11635        let hold = Arc::new(std::sync::Barrier::new(2));
11636        let handler = ControlHandler::with_forwarding(
11637            Arc::new(Registry::default()),
11638            Arc::new(ForwardingTable::default()),
11639        )
11640        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11641        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11642        let hello = {
11643            let handler = handler.clone();
11644            let module_ctx = module_ctx.clone();
11645            std::thread::spawn(move || {
11646                handler
11647                    .handle_hello(
11648                        module_ctx.connection_id,
11649                        Some(module_ctx.egress.clone()),
11650                        hello_frame("aft", PROTOCOL_VERSION, 7),
11651                    )
11652                    .unwrap()
11653            })
11654        };
11655        (handler, hold, hello, module_ctx, module_rx)
11656    }
11657
11658    /// While a HELLO has reserved its registry entry but its forwarding
11659    /// endpoint is not installed yet, `route.open` to the module can only be
11660    /// refused. `catalog.list` must not list the module in that window: a
11661    /// client that opens a route as soon as it sees a module listed and ready
11662    /// would be refused for a module the daemon had just advertised. The
11663    /// refusal in the window stays the retryable `target_unavailable`, never
11664    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
11665    /// Once the HELLO completes, the module is listed and a route to it opens.
11666    #[tokio::test]
11667    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11668        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11669        let client = ConnectionId::new(41);
11670        let (client_ctx, mut client_rx) = route_ctx(client);
11671
11672        hold.wait();
11673        // Parked: the registry holds "aft", forwarding has no endpoint for it.
11674        assert!(handler.registry.get_module("aft").unwrap().is_some());
11675        assert!(!handler
11676            .forwarding
11677            .has_live_module_connection("aft")
11678            .unwrap());
11679        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11680        assert!(
11681            listed.is_empty(),
11682            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11683        );
11684        let refused = handler
11685            .handle_control_frame(
11686                &client_ctx,
11687                route_open_frame(301, "aft", unique_project_root("listed-window")),
11688            )
11689            .await
11690            .unwrap();
11691        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11692        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11693        let duplicate = handler
11694            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11695            .await
11696            .unwrap();
11697        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11698
11699        hold.wait();
11700        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11701        let ack = module_rx.try_recv().unwrap().frame;
11702        assert_eq!(ack.header.ty, FrameType::HelloAck);
11703        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11704        assert_eq!(listed, vec![("aft".to_string(), true)]);
11705
11706        let (open_task, bind) = relay_route_open(
11707            &handler,
11708            client,
11709            &client_ctx.egress,
11710            &mut module_rx,
11711            303,
11712            "aft",
11713            "listed-after",
11714        )
11715        .await;
11716        assert!(handler
11717            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11718            .await
11719            .unwrap()
11720            .is_empty());
11721        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11722        published_route(&client_rx.try_recv().unwrap().frame);
11723    }
11724
11725    /// A catalog watcher keys on the generation: when it changes, it re-reads
11726    /// the listing. So the generation must advance exactly when the module
11727    /// appears. If it advanced while the module was still hidden, a watcher
11728    /// would read the new generation without the module, and the module would
11729    /// later appear under that same generation with nothing telling the
11730    /// watcher to look again.
11731    #[tokio::test]
11732    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11733        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11734        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11735        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11736
11737        hold.wait();
11738        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11739        assert!(listed.is_empty());
11740        assert_eq!(
11741            during, before,
11742            "the generation moved while the module was not listed"
11743        );
11744
11745        hold.wait();
11746        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11747        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11748        assert_eq!(listed, vec![("aft".to_string(), true)]);
11749        assert_eq!(
11750            after,
11751            before + 1,
11752            "the module appeared without exactly one generation bump"
11753        );
11754    }
11755
11756    /// A HELLO whose forwarding endpoint cannot be installed (here the
11757    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11758    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11759    /// and the catalog generation where it was.
11760    #[tokio::test]
11761    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11762        let registry = Arc::new(Registry::default());
11763        let handler = ControlHandler::with_forwarding(
11764            Arc::clone(&registry),
11765            Arc::new(ForwardingTable::default()),
11766        );
11767        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11768        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11769        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11770        drop(module_rx);
11771
11772        let replies = handler
11773            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11774            .await
11775            .unwrap();
11776        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11777        assert!(registry.get_module("aft").unwrap().is_none());
11778        assert!(registry
11779            .get_module_by_connection(module_ctx.connection_id)
11780            .unwrap()
11781            .is_none());
11782        assert_eq!(
11783            catalog_snapshot(&handler, &client_ctx, 301).await,
11784            (before, Vec::new())
11785        );
11786    }
11787
11788    /// A handler whose `route.open` parks after queueing the route.bind and
11789    /// before marking it enqueued, with module "aft" registered on connection
11790    /// 40. Returns the two-party barrier that parks and releases it.
11791    async fn route_open_held_after_bind_send() -> (
11792        ControlHandler,
11793        Arc<tokio::sync::Barrier>,
11794        RouteCtx,
11795        mpsc::Receiver<crate::router::OutboundFrame>,
11796    ) {
11797        let hold = Arc::new(tokio::sync::Barrier::new(2));
11798        let handler = ControlHandler::with_forwarding(
11799            Arc::new(Registry::default()),
11800            Arc::new(ForwardingTable::default()),
11801        )
11802        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11803        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11804        hello_via_sink(
11805            &handler,
11806            &module_ctx,
11807            &mut module_rx,
11808            hello_frame("aft", PROTOCOL_VERSION, 7),
11809        )
11810        .await;
11811        (handler, hold, module_ctx, module_rx)
11812    }
11813
11814    /// A fast module acks the route.bind, and the reader commits the route,
11815    /// before `route.open` has marked the relay enqueued. The daemon used to
11816    /// read the missing relay as an abandoned bind and send the module a
11817    /// GOODBYE for the route it had just committed: the client was told its
11818    /// route was open, the module unbound the channel, and the client's first
11819    /// request was dropped on the module side as unbound.
11820    #[tokio::test]
11821    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11822        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11823        let client = ConnectionId::new(41);
11824        let (client_ctx, mut client_rx) = route_ctx(client);
11825
11826        let (open_task, bind) = relay_route_open(
11827            &handler,
11828            client,
11829            &client_ctx.egress,
11830            &mut module_rx,
11831            200,
11832            "aft",
11833            "acked-before-mark",
11834        )
11835        .await;
11836        let (module_channel, module_epoch) = route_bind_channel(&bind);
11837        // route.open is now parked between queueing the bind and marking it.
11838        hold.wait().await;
11839
11840        // The module answers inside that window, and the route commits.
11841        assert!(handler
11842            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11843            .await
11844            .unwrap()
11845            .is_empty());
11846        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11847
11848        hold.wait().await;
11849        assert!(
11850            open_task.await.unwrap().is_empty(),
11851            "route.open answers accepted"
11852        );
11853
11854        // Every frame route.open could have sent is already queued, because it
11855        // has returned; the module must have been sent nothing.
11856        match module_rx.try_recv() {
11857            Err(mpsc::error::TryRecvError::Empty) => {}
11858            Ok(outbound) => panic!(
11859                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11860                outbound.frame.header.ty,
11861                outbound.frame.header.channel,
11862                outbound.frame.header.epoch
11863            ),
11864            Err(err) => panic!("module egress closed: {err}"),
11865        }
11866
11867        // The client's first request on the route is the next thing the module
11868        // reads, on the binding it just created.
11869        let router = Router::with_control_handler(Arc::new(handler.clone()));
11870        let request = Frame::build(
11871            FrameType::Request,
11872            Flags::new(true, Priority::Interactive, false),
11873            client_channel,
11874            client_epoch,
11875            201,
11876            b"first request".to_vec(),
11877        )
11878        .unwrap();
11879        router
11880            .route_for_connection(&client_ctx, request)
11881            .await
11882            .unwrap();
11883        let delivered = module_rx.try_recv().unwrap().frame;
11884        assert_eq!(delivered.header.ty, FrameType::Request);
11885        assert_eq!(
11886            (delivered.header.channel, delivered.header.epoch),
11887            (module_channel, module_epoch)
11888        );
11889        assert_eq!(delivered.body, b"first request");
11890    }
11891
11892    /// The client goes away while `route.open` is still queueing the bind, so
11893    /// the relay is dropped with no module reply. The teardown cannot send the
11894    /// GOODBYE itself (it could reach the module ahead of the bind), so
11895    /// `route.open` must send it once the bind is queued; otherwise the module
11896    /// keeps a binding the daemon will never commit.
11897    #[tokio::test]
11898    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11899        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11900        let client = ConnectionId::new(42);
11901        let (client_ctx, _client_rx) = route_ctx(client);
11902
11903        let (open_task, bind) = relay_route_open(
11904            &handler,
11905            client,
11906            &client_ctx.egress,
11907            &mut module_rx,
11908            300,
11909            "aft",
11910            "released-before-mark",
11911        )
11912        .await;
11913        let (module_channel, module_epoch) = route_bind_channel(&bind);
11914        hold.wait().await;
11915
11916        handler.cleanup_connection(client).unwrap();
11917        assert!(
11918            module_rx.try_recv().is_err(),
11919            "client teardown must not GOODBYE a bind route.open is still queueing"
11920        );
11921
11922        hold.wait().await;
11923        let response = open_task.await.unwrap();
11924        assert_eq!(response.len(), 1);
11925        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11926
11927        let goodbye = module_rx
11928            .try_recv()
11929            .expect("module is told to drop the binding the daemon abandoned")
11930            .frame;
11931        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11932        assert_eq!(
11933            (goodbye.header.channel, goodbye.header.epoch),
11934            (module_channel, module_epoch)
11935        );
11936        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11937    }
11938
11939    /// The module acks inside the same window, but the client's egress closed
11940    /// first, so the reader refuses to commit. The ack proves the module holds
11941    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11942    /// send a second one.
11943    #[tokio::test]
11944    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11945        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11946        let client = ConnectionId::new(43);
11947        let (client_ctx, client_rx) = route_ctx(client);
11948
11949        let (open_task, bind) = relay_route_open(
11950            &handler,
11951            client,
11952            &client_ctx.egress,
11953            &mut module_rx,
11954            400,
11955            "aft",
11956            "gone-client-before-mark",
11957        )
11958        .await;
11959        let (module_channel, module_epoch) = route_bind_channel(&bind);
11960        hold.wait().await;
11961
11962        drop(client_rx);
11963        assert!(client_ctx.egress.is_closed());
11964        handler
11965            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11966            .await
11967            .unwrap();
11968
11969        hold.wait().await;
11970        let response = open_task.await.unwrap();
11971        assert_eq!(response.len(), 1);
11972        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11973
11974        let goodbye = module_rx
11975            .try_recv()
11976            .expect("module is told to drop the binding it acked")
11977            .frame;
11978        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11979        assert_eq!(
11980            (goodbye.header.channel, goodbye.header.epoch),
11981            (module_channel, module_epoch)
11982        );
11983        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11984        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11985    }
11986
11987    /// The fence at the module-loop boundary, stated as its own contract: which
11988    /// forwarding failures are allowed to end the module connection that is being
11989    /// served. A `ConnectionClosing` naming some client is about that client, and
11990    /// a module connection is shared; the same error naming the module's own
11991    /// connection is about this connection and must stay fatal, as must failures
11992    /// that are about the forwarding table itself.
11993    #[test]
11994    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11995        let handler = ControlHandler::default();
11996        let module_connection = ConnectionId::new(30);
11997        let client_connection = ConnectionId::new(31);
11998
11999        handler
12000            .refuse_to_end_module_connection_for_a_client(
12001                module_connection,
12002                77,
12003                ForwardingError::ConnectionClosing {
12004                    connection_id: client_connection,
12005                },
12006            )
12007            .expect("a closing client must never end the module connection");
12008
12009        assert!(matches!(
12010            handler.refuse_to_end_module_connection_for_a_client(
12011                module_connection,
12012                78,
12013                ForwardingError::ConnectionClosing {
12014                    connection_id: module_connection,
12015                },
12016            ),
12017            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12018                connection_id
12019            })) if connection_id == module_connection
12020        ));
12021        assert!(matches!(
12022            handler.refuse_to_end_module_connection_for_a_client(
12023                module_connection,
12024                79,
12025                ForwardingError::Poisoned,
12026            ),
12027            Err(RouterError::Forwarding(ForwardingError::Poisoned))
12028        ));
12029        assert!(matches!(
12030            handler.refuse_to_end_module_connection_for_a_client(
12031                module_connection,
12032                80,
12033                ForwardingError::StaleModuleEndpoint,
12034            ),
12035            Err(RouterError::Forwarding(
12036                ForwardingError::StaleModuleEndpoint
12037            ))
12038        ));
12039    }
12040
12041    /// The spawn-attestation guard is what stops a connected module from claiming
12042    /// another module's identity and being stamped `Reserved` for it. Every other
12043    /// test that supplies a consumer_identity supplies a CORRECT one, because a
12044    /// correct one is what the rest of the flow needs -- so the guard's rejection
12045    /// branch was never the subject of an assertion, only its acceptance branch.
12046    ///
12047    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
12048    /// whole subc-core library suite green; only the forwarding integration tests
12049    /// notice, and they notice for unrelated reasons. This test exists so the
12050    /// refusal itself is asserted where the guard lives: it fails if the identity
12051    /// check stops refusing, which is the direction that matters, since a guard
12052    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
12053    #[tokio::test]
12054    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12055        let registry = Arc::new(Registry::default());
12056        let forwarding = Arc::new(ForwardingTable::default());
12057        let supervisor = SupervisorHandle::new();
12058        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12059        let handler =
12060            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12061                .with_supervisor(supervisor);
12062
12063        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12064        hello_via_sink(
12065            &handler,
12066            &target_ctx,
12067            &mut target_rx,
12068            hello_frame("target", PROTOCOL_VERSION, 1),
12069        )
12070        .await;
12071
12072        // A real supervised module id presenting the wrong nonce. This is the
12073        // impersonation case: the attacker knows a privileged module_id, which is
12074        // public, and guesses at the nonce, which is not.
12075        let wrong_nonce = handler
12076            .handle_control_frame(
12077                &route_ctx(ConnectionId::new(91)).0,
12078                route_open_frame_with_admission_facts(
12079                    20,
12080                    "target",
12081                    unique_project_root("admission-facts"),
12082                    Some(subc_control::ConsumerIdentity {
12083                        module_id: "fed".to_string(),
12084                        launch_nonce: "not-the-real-nonce".to_string(),
12085                    }),
12086                    None,
12087                ),
12088            )
12089            .await
12090            .unwrap();
12091        assert_eq!(
12092            parse_error(&wrong_nonce[0])["code"],
12093            "bad_consumer_identity",
12094            "a mismatched launch nonce must be refused, not stamped Reserved"
12095        );
12096
12097        // A module id the supervisor never spawned at all, so no nonce exists to
12098        // compare against. An implementation that treats "no record" as "nothing
12099        // to check" fails open here while passing the case above.
12100        let never_spawned = handler
12101            .handle_control_frame(
12102                &route_ctx(ConnectionId::new(92)).0,
12103                route_open_frame_with_admission_facts(
12104                    21,
12105                    "target",
12106                    unique_project_root("admission-facts"),
12107                    Some(subc_control::ConsumerIdentity {
12108                        module_id: "never-spawned".to_string(),
12109                        launch_nonce: "any-nonce".to_string(),
12110                    }),
12111                    None,
12112                ),
12113            )
12114            .await
12115            .unwrap();
12116        assert_eq!(
12117            parse_error(&never_spawned[0])["code"],
12118            "bad_consumer_identity",
12119            "an unspawned module_id must be refused rather than accepted for lack of a record"
12120        );
12121    }
12122
12123    /// The refusal test above proves the guard says NO. Nothing proved it can say
12124    /// YES, and the difference is not academic: replacing the whole authorization
12125    /// with `false` -- admitting no consumer identity at all, revoking Reserved
12126    /// standing for every supervised module in the fleet -- leaves 110 of the 111
12127    /// library tests GREEN. The one that notices does so by HANGING, because it
12128    /// waits for a bind that can no longer happen.
12129    ///
12130    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
12131    /// or flaky test, invites a RETRY rather than an investigation, and the retry
12132    /// hangs too and gets blamed on the runner. So a total revocation of the
12133    /// daemon's trust grant would have shipped behind a symptom nobody attributes
12134    /// to code.
12135    ///
12136    /// The bias is structural rather than accidental. A REFUSAL looks like a
12137    /// failure someone writes a test for; a GRANT looks like the happy path. Every
12138    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
12139    /// suite for that reason, and this one is the purest case in the daemon.
12140    ///
12141    /// This test asserts the EFFECT rather than the absence of an error: the module
12142    /// receives a RouteBind and it carries `Reserved` naming the attested module.
12143    /// A guard that admitted nobody would produce no bind at all; one that admitted
12144    /// everybody would stamp the wrong principal, which the refusal test catches.
12145    #[tokio::test]
12146    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12147        let registry = Arc::new(Registry::default());
12148        let forwarding = Arc::new(ForwardingTable::default());
12149        let supervisor = SupervisorHandle::new();
12150        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12151        let handler =
12152            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12153                .with_supervisor(supervisor);
12154
12155        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12156        hello_via_sink(
12157            &handler,
12158            &target_ctx,
12159            &mut target_rx,
12160            hello_frame("target", PROTOCOL_VERSION, 1),
12161        )
12162        .await;
12163
12164        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12165        let route_handler = handler.clone();
12166        let route_task = tokio::spawn(async move {
12167            route_handler
12168                .handle_control_frame(
12169                    &client_ctx,
12170                    route_open_frame_with_admission_facts(
12171                        30,
12172                        "target",
12173                        unique_project_root("admission-facts"),
12174                        Some(subc_control::ConsumerIdentity {
12175                            module_id: "fed".to_string(),
12176                            launch_nonce: "fed-nonce".to_string(),
12177                        }),
12178                        None,
12179                    ),
12180                )
12181                .await
12182                .unwrap()
12183        });
12184
12185        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
12186        // the very mutation it exists to catch -- a guard that admits nobody -- no
12187        // bind is ever sent, so it HUNG rather than failing. That reproduces the
12188        // exact defect being fixed: a total revocation detected only as a stalled
12189        // suite, which reads as flakiness and invites a retry. An acceptance test
12190        // that waits for an effect must bound the wait, or a red becomes a hang.
12191        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12192            .await
12193            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12194            .expect("module control channel closed before route.bind");
12195        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12196        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12197            panic!("expected route.bind")
12198        };
12199        assert_eq!(
12200            principal,
12201            Some(Principal::Reserved {
12202                module_id: "fed".to_string()
12203            }),
12204            "a correctly attested consumer must be stamped Reserved for its own id"
12205        );
12206
12207        handler
12208            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12209            .await
12210            .unwrap();
12211        assert!(route_task.await.unwrap().is_empty());
12212        assert!(
12213            matches!(
12214                serde_json::from_slice::<ClientControlResponse>(
12215                    &client_rx.recv().await.unwrap().body
12216                )
12217                .unwrap(),
12218                ClientControlResponse::RouteOpen { .. }
12219            ),
12220            "the route must actually open, not merely avoid an error"
12221        );
12222    }
12223
12224    #[tokio::test(start_paused = true)]
12225    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12226        let registry = Arc::new(Registry::default());
12227        let forwarding = Arc::new(ForwardingTable::default());
12228        let supervisor = SupervisorHandle::new();
12229        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12230        let handler =
12231            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12232                .with_supervisor(supervisor);
12233
12234        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12235        hello_via_sink(
12236            &handler,
12237            &target_ctx,
12238            &mut target_rx,
12239            hello_frame("target", PROTOCOL_VERSION, 1),
12240        )
12241        .await;
12242
12243        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12244        let direct_handler = handler.clone();
12245        let direct_open = tokio::spawn(async move {
12246            direct_handler
12247                .handle_control_frame(
12248                    &direct_ctx,
12249                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
12250                )
12251                .await
12252                .unwrap()
12253        });
12254        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12255            .await
12256            .expect("no direct route.bind within 5s")
12257            .expect("target control channel closed before direct route.bind");
12258        handler
12259            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12260            .await
12261            .unwrap();
12262        assert!(direct_open.await.unwrap().is_empty());
12263        let _ = direct_rx.recv().await.unwrap();
12264
12265        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12266        let reserved_handler = handler.clone();
12267        let reserved_open = tokio::spawn(async move {
12268            reserved_handler
12269                .handle_control_frame(
12270                    &reserved_ctx,
12271                    route_open_frame_with_admission_facts(
12272                        3,
12273                        "target",
12274                        unique_project_root("admission-facts"),
12275                        Some(ConsumerIdentity {
12276                            module_id: "fed".to_string(),
12277                            launch_nonce: "fed-nonce".to_string(),
12278                        }),
12279                        None,
12280                    ),
12281                )
12282                .await
12283                .unwrap()
12284        });
12285        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12286            .await
12287            .expect("no reserved route.bind within 5s")
12288            .expect("target control channel closed before reserved route.bind");
12289        handler
12290            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12291            .await
12292            .unwrap();
12293        assert!(reserved_open.await.unwrap().is_empty());
12294        let _ = reserved_rx.recv().await.unwrap();
12295
12296        forwarding
12297            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12298            .unwrap();
12299        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12300        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12301            module_id: Some("target".to_string()),
12302        })
12303        .unwrap();
12304        let census_frame =
12305            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12306        let response = handler
12307            .handle_control_frame(&census_ctx, census_frame)
12308            .await
12309            .unwrap()
12310            .pop()
12311            .unwrap();
12312        let actual: Value = serde_json::from_slice(&response.body).unwrap();
12313        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12314        assert!(matches!(
12315            decoded,
12316            ClientControlResponse::SupervisorRoutes { .. }
12317        ));
12318        let routes = actual["modules"][0]["routes"].as_array().unwrap();
12319        assert_eq!(routes.len(), 2);
12320        assert!(routes.iter().all(|route| route["draining"] == true));
12321        // The census carries WHY: the reason the drain was begun with, in the
12322        // route.closing vocabulary, on every draining route this drain marked.
12323        assert!(
12324            routes.iter().all(|route| route["drain_reason"] == "reload"),
12325            "draining routes must name the drain's reason: {routes:?}"
12326        );
12327        assert!(routes.iter().any(|route| {
12328            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12329        }));
12330        assert!(routes.iter().any(|route| {
12331            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12332        }));
12333
12334        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12335            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12336        if std::env::var_os("UPDATE_GOLDEN").is_some() {
12337            std::fs::write(
12338                &golden_path,
12339                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12340            )
12341            .unwrap();
12342        }
12343        let expected: Value =
12344            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12345        assert_eq!(actual, expected);
12346    }
12347
12348    async fn query_live_roots(
12349        handler: &ControlHandler,
12350        module_ctx: &RouteCtx,
12351    ) -> ModuleControlResponseToModule {
12352        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12353        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12354        let response = handler
12355            .handle_control_frame(module_ctx, frame)
12356            .await
12357            .unwrap()
12358            .pop()
12359            .unwrap();
12360        serde_json::from_slice(&response.body).unwrap()
12361    }
12362
12363    #[tokio::test(start_paused = true)]
12364    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12365        let registry = Arc::new(Registry::default());
12366        let forwarding = Arc::new(ForwardingTable::default());
12367        let handler = ControlHandler::with_forwarding(registry, forwarding);
12368        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12369        hello_via_sink(
12370            &handler,
12371            &target_ctx,
12372            &mut target_rx,
12373            hello_frame("target", PROTOCOL_VERSION, 1),
12374        )
12375        .await;
12376        let root = unique_project_root("live-roots-known");
12377        let path = ProjectRootId::from_path_allowing_missing(root.path())
12378            .unwrap()
12379            .as_path()
12380            .to_path_buf();
12381        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12382        let open_handler = handler.clone();
12383        let opened = tokio::spawn(async move {
12384            open_handler
12385                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12386                .await
12387                .unwrap()
12388        });
12389        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12390            .await
12391            .unwrap()
12392            .unwrap();
12393        handler
12394            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12395            .await
12396            .unwrap();
12397        assert!(opened.await.unwrap().is_empty());
12398        let _ = client_rx.recv().await.unwrap();
12399
12400        let root = unique_project_root("live-roots-pending");
12401        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12402            .unwrap()
12403            .as_path()
12404            .to_path_buf();
12405        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12406        let open_handler = handler.clone();
12407        let pending = tokio::spawn(async move {
12408            open_handler
12409                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12410                .await
12411                .unwrap()
12412        });
12413        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12414            .await
12415            .unwrap()
12416            .unwrap();
12417        let actual = query_live_roots(&handler, &target_ctx).await;
12418        let ModuleControlResponseToModule::LiveRoots {
12419            roots,
12420            unknown_root_bindings,
12421            total_bindings,
12422        } = actual
12423        else {
12424            panic!("expected live roots")
12425        };
12426        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12427        assert_eq!(unknown_root_bindings, 0);
12428        assert_eq!(
12429            roots.len(),
12430            2,
12431            "root-known arm must retain each canonical root"
12432        );
12433        assert_eq!(
12434            total_bindings,
12435            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12436        );
12437        let counts = roots
12438            .iter()
12439            .map(|root| (root.project_root.clone(), root.bound, root.pending))
12440            .collect::<Vec<_>>();
12441        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12442        expected.sort_by(|a, b| a.0.cmp(&b.0));
12443        assert_eq!(
12444            counts, expected,
12445            "roots must sort by path and count pending separately"
12446        );
12447        handler
12448            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12449            .await
12450            .unwrap();
12451        assert!(pending.await.unwrap().is_empty());
12452    }
12453
12454    #[tokio::test(start_paused = true)]
12455    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12456        let forwarding = Arc::new(ForwardingTable::default());
12457        let handler =
12458            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12459        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12460        hello_via_sink(
12461            &handler,
12462            &target_ctx,
12463            &mut target_rx,
12464            hello_frame("target", PROTOCOL_VERSION, 1),
12465        )
12466        .await;
12467        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12468        let pending = forwarding
12469            .begin_route_bind_relay_for_test(
12470                client_ctx.connection_id,
12471                client_ctx.egress.clone(),
12472                2,
12473                "target",
12474            )
12475            .unwrap();
12476        forwarding
12477            .complete_pending_relay(
12478                target_ctx.connection_id,
12479                pending.corr,
12480                RouteBindRelayOutcome::Accepted,
12481            )
12482            .unwrap();
12483        let actual = query_live_roots(&handler, &target_ctx).await;
12484        let ModuleControlResponseToModule::LiveRoots {
12485            roots,
12486            unknown_root_bindings,
12487            total_bindings,
12488        } = actual
12489        else {
12490            panic!("expected live roots")
12491        };
12492        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12493        assert_eq!(
12494            unknown_root_bindings, 1,
12495            "unknown-root arm must not read as no bindings"
12496        );
12497        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12498        assert_eq!(
12499            total_bindings,
12500            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12501        );
12502    }
12503
12504    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12505    /// so the ack has to be on its outbound queue before the module is
12506    /// routable. The connection loop writes a handler's replies only after the
12507    /// handler returns; this test stops in exactly that gap, runs a real
12508    /// route.open from another connection, and only then writes whatever the
12509    /// HELLO handler returned, the way the loop would. If the ack were still a
12510    /// reply, the route.bind request would reach the module first.
12511    #[tokio::test(start_paused = true)]
12512    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12513        let forwarding = Arc::new(ForwardingTable::default());
12514        let handler =
12515            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12516        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12517        let replies = handler
12518            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12519            .await
12520            .unwrap();
12521        let queued_by_hello = module_rx.len();
12522
12523        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12524        let open_handler = handler.clone();
12525        let open = tokio::spawn(async move {
12526            open_handler
12527                .handle_control_frame(
12528                    &client_ctx,
12529                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12530                )
12531                .await
12532                .unwrap()
12533        });
12534        // Let the route.open run until its route.bind is on the module's queue.
12535        let mut spins = 0;
12536        while module_rx.len() == queued_by_hello {
12537            spins += 1;
12538            assert!(spins < 10_000, "route.open never queued a route.bind");
12539            tokio::task::yield_now().await;
12540        }
12541
12542        // Now the connection loop's half: write the HELLO handler's replies.
12543        for reply in replies {
12544            module_ctx.egress.send(reply).await.unwrap();
12545        }
12546
12547        let first = module_rx.recv().await.unwrap().frame;
12548        assert_eq!(
12549            first.header.ty,
12550            FrameType::HelloAck,
12551            "the first frame a registering module reads must be its HELLO_ACK"
12552        );
12553        assert_eq!(first.header.corr, 7);
12554        let second = module_rx.recv().await.unwrap().frame;
12555        assert_eq!(second.header.ty, FrameType::Request);
12556        assert!(
12557            matches!(
12558                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12559                ModuleControlRequest::RouteBind { .. }
12560            ),
12561            "the route.bind follows the ack"
12562        );
12563        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12564
12565        handler
12566            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12567            .await
12568            .unwrap();
12569        assert!(open.await.unwrap().is_empty());
12570        let _ = client_rx.recv().await.unwrap();
12571    }
12572
12573    #[tokio::test(start_paused = true)]
12574    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12575        let handler = ControlHandler::with_forwarding(
12576            Arc::new(Registry::default()),
12577            Arc::new(ForwardingTable::default()),
12578        );
12579        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12580        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12581        hello_via_sink(
12582            &handler,
12583            &first_ctx,
12584            &mut first_rx,
12585            hello_frame("first", PROTOCOL_VERSION, 1),
12586        )
12587        .await;
12588        hello_via_sink(
12589            &handler,
12590            &second_ctx,
12591            &mut second_rx,
12592            hello_frame("second", PROTOCOL_VERSION, 2),
12593        )
12594        .await;
12595        let root = unique_project_root("second-only");
12596        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12597        let cloned = handler.clone();
12598        let open = tokio::spawn(async move {
12599            cloned
12600                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12601                .await
12602                .unwrap()
12603        });
12604        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12605            .await
12606            .unwrap()
12607            .unwrap();
12608        let first = query_live_roots(&handler, &first_ctx).await;
12609        let second = query_live_roots(&handler, &second_ctx).await;
12610        assert!(
12611            matches!(
12612                first,
12613                ModuleControlResponseToModule::LiveRoots {
12614                    total_bindings: 0,
12615                    ..
12616                }
12617            ),
12618            "cross-module scope must not expose another module's roots"
12619        );
12620        assert!(
12621            matches!(
12622                second,
12623                ModuleControlResponseToModule::LiveRoots {
12624                    total_bindings: 1,
12625                    ..
12626                }
12627            ),
12628            "second module must see its pending route"
12629        );
12630        handler
12631            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12632            .await
12633            .unwrap();
12634        assert!(open.await.unwrap().is_empty());
12635    }
12636
12637    #[tokio::test(start_paused = true)]
12638    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12639        let handler = ControlHandler::with_forwarding(
12640            Arc::new(Registry::default()),
12641            Arc::new(ForwardingTable::default()),
12642        );
12643        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12644        hello_via_sink(
12645            &handler,
12646            &target_ctx,
12647            &mut target_rx,
12648            hello_frame("target", PROTOCOL_VERSION, 1),
12649        )
12650        .await;
12651        let actual = query_live_roots(&handler, &target_ctx).await;
12652        let ModuleControlResponseToModule::LiveRoots {
12653            roots,
12654            unknown_root_bindings,
12655            total_bindings,
12656        } = actual
12657        else {
12658            panic!("expected live roots")
12659        };
12660        assert!(roots.is_empty());
12661        assert_eq!(unknown_root_bindings, 0);
12662        assert_eq!(total_bindings, 0);
12663        assert_eq!(
12664            total_bindings,
12665            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12666        );
12667    }
12668
12669    /// Read the vendored fed corpus rather than hand-building a package.
12670    ///
12671    /// A hand-built object encodes what the test author believed the carrier
12672    /// emits. These vectors are what it actually emits, and one of them exists
12673    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
12674    /// ignores additive unknown fields at the traversal emit terminus."
12675    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12676        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12677            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12678        let text = std::fs::read_to_string(&path)
12679            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12680        let vectors: Vec<(String, Value)> = text
12681            .lines()
12682            .filter(|line| !line.trim().is_empty())
12683            .map(|line| {
12684                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12685                let id = entry["corpus_id"]
12686                    .as_str()
12687                    .expect("every vector carries a corpus_id")
12688                    .to_string();
12689                (id, entry["package"].clone())
12690            })
12691            .collect();
12692        // Pin the count: a corpus that silently shrinks would take its coverage
12693        // with it, and a suite reading N-1 vectors reports the same clean pass
12694        // as one reading N.
12695        assert_eq!(
12696            vectors.len(),
12697            3,
12698            "vendored fed corpus changed size; re-sync from subc-federation"
12699        );
12700
12701        // Pin what makes the corpus DISCRIMINATING, not just present.
12702        //
12703        // The relay test below takes its expected value from the corpus, so the
12704        // corpus supplies the test's power to detect a lossy relay rather than
12705        // its correctness. A relay that dropped unrecognised fields would still
12706        // be caught -- but only by a package carrying fields it does not know.
12707        // Shrink every package to the handful of keys any implementation would
12708        // recognise and the test keeps passing over an input that can no longer
12709        // fail, which is the same clean green as a corpus that shrank away.
12710        //
12711        // So assert the precondition rather than duplicating the packages here:
12712        // at least one vector must carry a field beyond the small common set.
12713        // That is one claim to maintain instead of nine, and it fails loudly if
12714        // a re-sync ever flattens the corpus.
12715        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12716        let richest = vectors
12717            .iter()
12718            .filter_map(|(_, package)| package.as_object())
12719            .map(|object| {
12720                object
12721                    .keys()
12722                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12723                    .count()
12724            })
12725            .max()
12726            .unwrap_or(0);
12727        assert!(
12728            richest >= 2,
12729            "vendored corpus no longer carries a package with unmodelled fields, \
12730             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12731        );
12732
12733        vectors
12734    }
12735
12736    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12737    ///
12738    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12739    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12740    /// three-key object, so a relay that quietly dropped fields it did not
12741    /// recognise would satisfy it. These vectors carry nine keys including ones
12742    /// this crate has no type for, so a typed relay fails here and only here.
12743    #[tokio::test]
12744    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12745        for (corpus_id, package) in fed_admission_facts_vectors() {
12746            let registry = Arc::new(Registry::default());
12747            let forwarding = Arc::new(ForwardingTable::default());
12748            let supervisor = SupervisorHandle::new();
12749            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12750            let handler =
12751                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12752                    .with_supervisor(supervisor)
12753                    .with_admission_facts_config(
12754                        Some("fed".to_string()),
12755                        Some(vec!["target".to_string()]),
12756                    );
12757
12758            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12759            hello_via_sink(
12760                &handler,
12761                &target_ctx,
12762                &mut target_rx,
12763                hello_frame("target", PROTOCOL_VERSION, 1),
12764            )
12765            .await;
12766
12767            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12768            let route_handler = handler.clone();
12769            let expected = package.clone();
12770            let route_task = tokio::spawn(async move {
12771                route_handler
12772                    .handle_control_frame(
12773                        &client_ctx,
12774                        route_open_frame_with_admission_facts(
12775                            20,
12776                            "target",
12777                            unique_project_root("admission-facts"),
12778                            Some(subc_control::ConsumerIdentity {
12779                                module_id: "fed".to_string(),
12780                                launch_nonce: "fed-nonce".to_string(),
12781                            }),
12782                            Some(package),
12783                        ),
12784                    )
12785                    .await
12786                    .unwrap()
12787            });
12788
12789            let bind_frame = target_rx.recv().await.unwrap();
12790            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12791            let ModuleControlRequest::RouteBind {
12792                admission_facts, ..
12793            } = bind
12794            else {
12795                panic!("{corpus_id}: expected route.bind")
12796            };
12797            assert_eq!(
12798                admission_facts,
12799                Some(expected),
12800                "{corpus_id}: relay must not add, drop or reshape any field"
12801            );
12802
12803            handler
12804                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12805                .await
12806                .unwrap();
12807            route_task.await.unwrap();
12808        }
12809    }
12810
12811    #[tokio::test]
12812    async fn route_bind_never_relays_the_openers_build_class() {
12813        let registry = Arc::new(Registry::default());
12814        let forwarding = Arc::new(ForwardingTable::default());
12815        let handler =
12816            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12817        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12818        hello_via_sink(
12819            &handler,
12820            &target_ctx,
12821            &mut target_rx,
12822            hello_frame("target", PROTOCOL_VERSION, 1),
12823        )
12824        .await;
12825
12826        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
12827        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12828        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12829            busy_refusals: None,
12830            target: RouteTarget::ToolProvider {
12831                module_id: "target".to_string(),
12832            },
12833            identity,
12834            consumer_identity: None,
12835            consumer_capabilities: None,
12836            role_versions: None,
12837            admission_facts: None,
12838            scope: None,
12839        })
12840        .unwrap();
12841        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12842
12843        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12844        let route_handler = handler.clone();
12845        let route_task = tokio::spawn(async move {
12846            route_handler
12847                .handle_control_frame(&client_ctx, open)
12848                .await
12849                .unwrap()
12850        });
12851        let bind_frame = target_rx.recv().await.unwrap();
12852        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12853        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12854            panic!("expected route.bind")
12855        };
12856        assert_eq!(
12857            identity.build_class, None,
12858            "route.bind carried the build class"
12859        );
12860        assert!(
12861            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12862            "the route.bind bytes still name build_class"
12863        );
12864        handler
12865            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12866            .await
12867            .unwrap();
12868        route_task.await.unwrap();
12869    }
12870
12871    #[tokio::test]
12872    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12873        let registry = Arc::new(Registry::default());
12874        let forwarding = Arc::new(ForwardingTable::default());
12875        let supervisor = SupervisorHandle::new();
12876        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12877        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12878        let handler =
12879            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12880                .with_supervisor(supervisor)
12881                .with_admission_facts_config(
12882                    Some("fed".to_string()),
12883                    Some(vec!["target".to_string()]),
12884                );
12885
12886        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12887        hello_via_sink(
12888            &handler,
12889            &target_ctx,
12890            &mut target_rx,
12891            hello_frame("target", PROTOCOL_VERSION, 1),
12892        )
12893        .await;
12894        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12895        hello_via_sink(
12896            &handler,
12897            &other_ctx,
12898            &mut other_rx,
12899            hello_frame("other", PROTOCOL_VERSION, 2),
12900        )
12901        .await;
12902
12903        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12904        let expected_facts = facts.clone();
12905        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12906        let route_handler = handler.clone();
12907        let route_task = tokio::spawn(async move {
12908            route_handler
12909                .handle_control_frame(
12910                    &client_ctx,
12911                    route_open_frame_with_admission_facts(
12912                        10,
12913                        "target",
12914                        unique_project_root("admission-facts"),
12915                        Some(subc_control::ConsumerIdentity {
12916                            module_id: "fed".to_string(),
12917                            launch_nonce: "fed-nonce".to_string(),
12918                        }),
12919                        Some(facts.clone()),
12920                    ),
12921                )
12922                .await
12923                .unwrap()
12924        });
12925        let bind_frame = target_rx.recv().await.unwrap();
12926        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12927        let ModuleControlRequest::RouteBind {
12928            admission_facts, ..
12929        } = bind
12930        else {
12931            panic!("expected route.bind")
12932        };
12933        assert_eq!(admission_facts, Some(expected_facts));
12934        handler
12935            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12936            .await
12937            .unwrap();
12938        assert!(route_task.await.unwrap().is_empty());
12939        assert!(matches!(
12940            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12941                .unwrap(),
12942            ClientControlResponse::RouteOpen { .. }
12943        ));
12944
12945        let direct = handler
12946            .handle_control_frame(
12947                &route_ctx(ConnectionId::new(73)).0,
12948                route_open_frame_with_admission_facts(
12949                    11,
12950                    "target",
12951                    unique_project_root("admission-facts"),
12952                    None,
12953                    Some(json!({"x": 1})),
12954                ),
12955            )
12956            .await
12957            .unwrap();
12958        assert_eq!(
12959            parse_error(&direct[0])["code"],
12960            "admission_facts_not_permitted"
12961        );
12962
12963        let different_reserved = handler
12964            .handle_control_frame(
12965                &route_ctx(ConnectionId::new(77)).0,
12966                route_open_frame_with_admission_facts(
12967                    15,
12968                    "target",
12969                    unique_project_root("admission-facts"),
12970                    Some(subc_control::ConsumerIdentity {
12971                        module_id: "other".to_string(),
12972                        launch_nonce: "other-nonce".to_string(),
12973                    }),
12974                    Some(json!({"x": 1})),
12975                ),
12976            )
12977            .await
12978            .unwrap();
12979        assert_eq!(
12980            parse_error(&different_reserved[0])["code"],
12981            "admission_facts_not_permitted"
12982        );
12983
12984        let other_target = handler
12985            .handle_control_frame(
12986                &route_ctx(ConnectionId::new(74)).0,
12987                route_open_frame_with_admission_facts(
12988                    12,
12989                    "other",
12990                    unique_project_root("admission-facts"),
12991                    Some(subc_control::ConsumerIdentity {
12992                        module_id: "fed".to_string(),
12993                        launch_nonce: "fed-nonce".to_string(),
12994                    }),
12995                    Some(json!({"x": 1})),
12996                ),
12997            )
12998            .await
12999            .unwrap();
13000        assert_eq!(
13001            parse_error(&other_target[0])["code"],
13002            "admission_facts_target_not_allowed"
13003        );
13004
13005        let nonexistent = handler
13006            .handle_control_frame(
13007                &route_ctx(ConnectionId::new(75)).0,
13008                route_open_frame_with_admission_facts(
13009                    13,
13010                    "missing",
13011                    unique_project_root("admission-facts"),
13012                    None,
13013                    Some(json!({"x": 1})),
13014                ),
13015            )
13016            .await
13017            .unwrap();
13018        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13019
13020        let described = handler
13021            .handle_control_frame(
13022                &route_ctx(ConnectionId::new(76)).0,
13023                Frame::build(
13024                    FrameType::Request,
13025                    control_flags(),
13026                    0,
13027                    0,
13028                    14,
13029                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13030                )
13031                .unwrap(),
13032            )
13033            .await
13034            .unwrap();
13035        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13036            serde_json::from_slice(&described[0].body).unwrap()
13037        else {
13038            panic!("expected server.describe response")
13039        };
13040        assert!(capabilities
13041            .iter()
13042            .any(|cap| cap == "admission_facts_relay_v1"));
13043    }
13044
13045    #[tokio::test]
13046    async fn admission_facts_without_configured_carrier_are_rejected() {
13047        let registry = Arc::new(Registry::default());
13048        let forwarding = Arc::new(ForwardingTable::default());
13049        let handler = ControlHandler::with_forwarding(registry, forwarding);
13050        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13051        hello_via_sink(
13052            &handler,
13053            &target_ctx,
13054            &mut target_rx,
13055            hello_frame("target", PROTOCOL_VERSION, 1),
13056        )
13057        .await;
13058
13059        let responses = handler
13060            .handle_control_frame(
13061                &route_ctx(ConnectionId::new(79)).0,
13062                route_open_frame_with_admission_facts(
13063                    16,
13064                    "target",
13065                    unique_project_root("admission-facts"),
13066                    None,
13067                    Some(json!({"x": 1})),
13068                ),
13069            )
13070            .await
13071            .unwrap();
13072        assert_eq!(
13073            parse_error(&responses[0])["code"],
13074            "admission_facts_not_permitted"
13075        );
13076    }
13077
13078    #[tokio::test]
13079    async fn route_open_relays_consumer_capabilities_verbatim() {
13080        let registry = Arc::new(Registry::default());
13081        let forwarding = Arc::new(ForwardingTable::default());
13082        let handler =
13083            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13084        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13085        hello_via_sink(
13086            &handler,
13087            &module_ctx,
13088            &mut module_rx,
13089            hello_frame("aft", PROTOCOL_VERSION, 7),
13090        )
13091        .await;
13092
13093        let expected = vec!["elicitation".to_string(), "roots".to_string()];
13094        let expected_for_request = expected.clone();
13095        let project_root = unique_project_root("consumer-capabilities-present");
13096        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13097        let route_handler = handler.clone();
13098        let route_task = tokio::spawn(async move {
13099            route_handler
13100                .handle_control_frame(
13101                    &client_ctx,
13102                    route_open_frame_with_consumer_capabilities(
13103                        401,
13104                        "aft",
13105                        project_root,
13106                        Some(expected_for_request),
13107                    ),
13108                )
13109                .await
13110                .unwrap()
13111        });
13112        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13113            .await
13114            .unwrap()
13115            .unwrap();
13116        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13117        let ModuleControlRequest::RouteBind {
13118            consumer_capabilities,
13119            ..
13120        } = bind
13121        else {
13122            panic!("expected route.bind request, got {bind:?}");
13123        };
13124        assert_eq!(consumer_capabilities, Some(expected.clone()));
13125
13126        handler
13127            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13128            .await
13129            .unwrap();
13130        let route_response = route_task.await.unwrap();
13131        assert!(route_response.is_empty());
13132        let published = client_rx.recv().await.unwrap();
13133        assert!(matches!(
13134            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13135            ClientControlResponse::RouteOpen { .. }
13136        ));
13137    }
13138
13139    #[tokio::test]
13140    async fn route_open_without_consumer_capabilities_relays_none() {
13141        let registry = Arc::new(Registry::default());
13142        let forwarding = Arc::new(ForwardingTable::default());
13143        let handler =
13144            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13145        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13146        hello_via_sink(
13147            &handler,
13148            &module_ctx,
13149            &mut module_rx,
13150            hello_frame("aft", PROTOCOL_VERSION, 7),
13151        )
13152        .await;
13153
13154        let project_root = unique_project_root("consumer-capabilities-absent");
13155        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13156        let route_handler = handler.clone();
13157        let route_task = tokio::spawn(async move {
13158            route_handler
13159                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13160                .await
13161                .unwrap()
13162        });
13163        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13164            .await
13165            .unwrap()
13166            .unwrap();
13167        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13168        let ModuleControlRequest::RouteBind {
13169            consumer_capabilities,
13170            ..
13171        } = bind
13172        else {
13173            panic!("expected route.bind request, got {bind:?}");
13174        };
13175        assert_eq!(consumer_capabilities, None);
13176
13177        handler
13178            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13179            .await
13180            .unwrap();
13181        let route_response = route_task.await.unwrap();
13182        assert!(route_response.is_empty());
13183        let published = client_rx.recv().await.unwrap();
13184        assert!(matches!(
13185            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13186            ClientControlResponse::RouteOpen { .. }
13187        ));
13188    }
13189
13190    /// Opens a route to a freshly registered `aft` with `sent` as the
13191    /// route.open's role_versions, acks the bind, and returns the role_versions
13192    /// the module's bind carried.
13193    async fn bind_role_versions_for(
13194        sent: Option<BTreeMap<String, String>>,
13195        connection: u64,
13196    ) -> Option<BTreeMap<String, String>> {
13197        let registry = Arc::new(Registry::default());
13198        let forwarding = Arc::new(ForwardingTable::default());
13199        let handler =
13200            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13201        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13202        hello_via_sink(
13203            &handler,
13204            &module_ctx,
13205            &mut module_rx,
13206            hello_frame("aft", PROTOCOL_VERSION, 7),
13207        )
13208        .await;
13209        let project_root = unique_project_root("role-versions");
13210        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13211        let route_handler = handler.clone();
13212        let route_task = tokio::spawn(async move {
13213            route_handler
13214                .handle_control_frame(
13215                    &client_ctx,
13216                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
13217                )
13218                .await
13219                .unwrap()
13220        });
13221        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13222            .await
13223            .expect("a well-formed route.open reaches the module as a bind")
13224            .unwrap();
13225        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13226        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13227            panic!("expected route.bind request, got {bind:?}");
13228        };
13229        handler
13230            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13231            .await
13232            .unwrap();
13233        assert!(route_task.await.unwrap().is_empty());
13234        let published = client_rx.recv().await.unwrap();
13235        assert!(matches!(
13236            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13237            ClientControlResponse::RouteOpen { .. }
13238        ));
13239        role_versions
13240    }
13241
13242    #[tokio::test]
13243    async fn route_open_relays_role_versions_verbatim() {
13244        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13245        assert_eq!(
13246            bind_role_versions_for(Some(sent.clone()), 141).await,
13247            Some(sent)
13248        );
13249    }
13250
13251    /// An empty map declares nothing, so the provider sees no field rather
13252    /// than an empty object it would have to treat as a second "none".
13253    #[tokio::test]
13254    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13255        assert_eq!(bind_role_versions_for(None, 143).await, None);
13256        assert_eq!(
13257            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13258            None
13259        );
13260    }
13261
13262    #[tokio::test]
13263    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13264        let registry = Arc::new(Registry::default());
13265        let forwarding = Arc::new(ForwardingTable::default());
13266        let handler =
13267            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13268        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13269        hello_via_sink(
13270            &handler,
13271            &module_ctx,
13272            &mut module_rx,
13273            hello_frame("aft", PROTOCOL_VERSION, 7),
13274        )
13275        .await;
13276
13277        let nine: BTreeMap<String, String> = (0..9)
13278            .map(|index| (format!("role-{index}"), "v1".to_string()))
13279            .collect();
13280        for (label, malformed) in [
13281            (
13282                "invalid role name",
13283                role_versions(&[("Tool_Provider", "v1")]),
13284            ),
13285            ("invalid version", role_versions(&[("tool-provider", "v0")])),
13286            ("nine entries", nine),
13287        ] {
13288            // A refused open answers at once; one that reached the module
13289            // would wait for its bind ack and trip this timeout.
13290            let responses = tokio::time::timeout(
13291                Duration::from_secs(1),
13292                handler.handle_control_frame(
13293                    &route_ctx(ConnectionId::new(148)).0,
13294                    route_open_frame_with_role_versions(
13295                        404,
13296                        "aft",
13297                        unique_project_root("role-versions-malformed"),
13298                        Some(malformed),
13299                    ),
13300                ),
13301            )
13302            .await
13303            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13304            .unwrap();
13305            assert_eq!(responses.len(), 1, "{label}");
13306            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13307            let error = parse_error(&responses[0]);
13308            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13309            assert_eq!(
13310                error["detail"]["field"], "role_versions",
13311                "{label}: {error}"
13312            );
13313            assert!(
13314                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13315                "{label}: a malformed declaration is terminal"
13316            );
13317            assert!(
13318                module_rx.try_recv().is_err(),
13319                "{label}: the module must never see a bind"
13320            );
13321        }
13322    }
13323
13324    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
13325    /// consumer can tell this daemon forwards the field from one that would
13326    /// drop it.
13327    #[tokio::test]
13328    async fn route_role_versions_capability_is_advertised() {
13329        let handler = ControlHandler::new(Arc::new(Registry::default()));
13330        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13331        let ack = hello_via_sink(
13332            &handler,
13333            &ctx,
13334            &mut rx,
13335            hello_frame("m", PROTOCOL_VERSION, 1),
13336        )
13337        .await;
13338        let ack = parse_ack(&ack);
13339        assert!(
13340            ack.subc_capabilities
13341                .iter()
13342                .any(|c| c == "route-role-versions/v1"),
13343            "{:?}",
13344            ack.subc_capabilities
13345        );
13346
13347        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13348        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13349        let reply = handler
13350            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13351            .await
13352            .unwrap()
13353            .pop()
13354            .unwrap();
13355        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13356            serde_json::from_slice(&reply.body).unwrap()
13357        else {
13358            panic!("not a server.describe reply");
13359        };
13360        assert!(
13361            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13362            "{capabilities:?}"
13363        );
13364    }
13365
13366    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
13367    /// catalog entry without `events` (the module declares none) from a daemon
13368    /// that never reports declared events.
13369    #[tokio::test]
13370    async fn catalog_events_capability_is_advertised() {
13371        let handler = ControlHandler::new(Arc::new(Registry::default()));
13372        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13373        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13374        let reply = handler
13375            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13376            .await
13377            .unwrap()
13378            .pop()
13379            .unwrap();
13380        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13381            serde_json::from_slice(&reply.body).unwrap()
13382        else {
13383            panic!("not a server.describe reply");
13384        };
13385        assert!(
13386            capabilities
13387                .iter()
13388                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13389            "{capabilities:?}"
13390        );
13391    }
13392
13393    #[tokio::test]
13394    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13395        let handler = ControlHandler::new(Arc::new(Registry::default()));
13396        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13397        let ack = hello_via_sink(
13398            &handler,
13399            &module,
13400            &mut module_rx,
13401            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13402        )
13403        .await;
13404        let ack = parse_ack(&ack);
13405        assert!(
13406            ack.subc_capabilities
13407                .iter()
13408                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13409            "{:?}",
13410            ack.subc_capabilities
13411        );
13412        assert!(ack
13413            .subc_capabilities
13414            .iter()
13415            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13416
13417        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13418        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13419        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13420        let reply = handler
13421            .handle_control_frame(&client, frame)
13422            .await
13423            .unwrap()
13424            .pop()
13425            .unwrap();
13426        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13427            serde_json::from_slice(&reply.body).unwrap()
13428        else {
13429            panic!("not a server.describe reply");
13430        };
13431        assert!(
13432            capabilities
13433                .iter()
13434                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13435            "{capabilities:?}"
13436        );
13437        assert!(capabilities
13438            .iter()
13439            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13440    }
13441
13442    #[tokio::test]
13443    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13444        let handler = ControlHandler::new(Arc::new(Registry::default()));
13445        let (module, mut rx) = route_ctx(ConnectionId::new(156));
13446        let ack = hello_via_sink(
13447            &handler,
13448            &module,
13449            &mut rx,
13450            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13451        )
13452        .await;
13453        assert!(parse_ack(&ack)
13454            .subc_capabilities
13455            .iter()
13456            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13457        assert!(parse_ack(&ack)
13458            .subc_capabilities
13459            .iter()
13460            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13461        let (client, _rx) = route_ctx(ConnectionId::new(157));
13462        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13463        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13464        let reply = handler
13465            .handle_control_frame(&client, frame)
13466            .await
13467            .unwrap()
13468            .pop()
13469            .unwrap();
13470        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13471            serde_json::from_slice(&reply.body).unwrap()
13472        else {
13473            panic!("server.describe reply")
13474        };
13475        assert!(capabilities
13476            .iter()
13477            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13478        assert!(capabilities
13479            .iter()
13480            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13481    }
13482
13483    #[tokio::test]
13484    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13485        let handler = ControlHandler::new(Arc::new(Registry::default()));
13486        let (module, mut rx) = route_ctx(ConnectionId::new(158));
13487        let ack = hello_via_sink(
13488            &handler,
13489            &module,
13490            &mut rx,
13491            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13492        )
13493        .await;
13494        assert!(parse_ack(&ack)
13495            .subc_capabilities
13496            .iter()
13497            .any(|cap| cap == "scope-workdir/v1"));
13498        let (client, _rx) = route_ctx(ConnectionId::new(159));
13499        let frame = Frame::build(
13500            FrameType::Request,
13501            control_flags(),
13502            0,
13503            0,
13504            5,
13505            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13506        )
13507        .unwrap();
13508        let reply = handler
13509            .handle_control_frame(&client, frame)
13510            .await
13511            .unwrap()
13512            .remove(0);
13513        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13514            serde_json::from_slice(&reply.body).unwrap()
13515        else {
13516            panic!("server.describe reply")
13517        };
13518        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13519    }
13520
13521    #[tokio::test]
13522    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13523        let handler = ControlHandler::new(Arc::new(Registry::default()));
13524        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13525        let ack = hello_via_sink(
13526            &handler,
13527            &module,
13528            &mut rx,
13529            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13530        )
13531        .await;
13532        assert!(parse_ack(&ack)
13533            .subc_capabilities
13534            .iter()
13535            .any(|cap| cap == "worker-runs/v1"));
13536        let (client, _rx) = route_ctx(ConnectionId::new(161));
13537        let frame = Frame::build(
13538            FrameType::Request,
13539            control_flags(),
13540            0,
13541            0,
13542            5,
13543            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13544        )
13545        .unwrap();
13546        let reply = handler
13547            .handle_control_frame(&client, frame)
13548            .await
13549            .unwrap()
13550            .remove(0);
13551        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13552            serde_json::from_slice(&reply.body).unwrap()
13553        else {
13554            panic!("server.describe reply")
13555        };
13556        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13557    }
13558
13559    #[tokio::test]
13560    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13561        let handler = ControlHandler::new(Arc::new(Registry::default()));
13562        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13563        let ack = hello_via_sink(
13564            &handler,
13565            &module,
13566            &mut rx,
13567            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13568        )
13569        .await;
13570        assert!(parse_ack(&ack)
13571            .subc_capabilities
13572            .iter()
13573            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13574        let (client, _rx) = route_ctx(ConnectionId::new(155));
13575        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13576        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13577        let reply = handler
13578            .handle_control_frame(&client, frame)
13579            .await
13580            .unwrap()
13581            .pop()
13582            .unwrap();
13583        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13584            serde_json::from_slice(&reply.body).unwrap()
13585        else {
13586            panic!("not a server.describe reply")
13587        };
13588        assert!(capabilities
13589            .iter()
13590            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13591    }
13592
13593    #[tokio::test]
13594    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13595        let registry = Arc::new(Registry::default());
13596        let forwarding = Arc::new(ForwardingTable::default());
13597        let handler =
13598            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13599                .with_health_probe_timeout(Duration::from_secs(5));
13600        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13601        hello_via_sink(
13602            &handler,
13603            &module_ctx,
13604            &mut module_rx,
13605            non_routable_hello_frame_with_control_ops(
13606                "mcp",
13607                300,
13608                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13609            ),
13610        )
13611        .await;
13612        assert!(registry
13613            .get_module("mcp")
13614            .unwrap()
13615            .unwrap()
13616            .manifest
13617            .provides
13618            .is_empty());
13619
13620        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13621        let route_response = handler
13622            .handle_control_frame(
13623                &route_client_ctx,
13624                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13625            )
13626            .await
13627            .unwrap();
13628        assert_eq!(route_response[0].header.ty, FrameType::Error);
13629        assert_eq!(
13630            parse_error(&route_response[0])["code"],
13631            "target_unavailable"
13632        );
13633        assert!(parse_error(&route_response[0])["message"]
13634            .as_str()
13635            .unwrap()
13636            .contains("does not provide the requested target"));
13637        assert!(module_rx.try_recv().is_err());
13638
13639        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13640        let health_handler = handler.clone();
13641        let health_task = tokio::spawn(async move {
13642            health_handler
13643                .handle_control_frame(
13644                    &health_client_ctx,
13645                    supervisor_health_probe_frame(302, "mcp"),
13646                )
13647                .await
13648                .unwrap()
13649        });
13650        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13651            .await
13652            .unwrap()
13653            .unwrap();
13654        assert_eq!(
13655            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13656            ModuleControlRequest::HealthCheck {}
13657        );
13658        handler
13659            .handle_control_frame(
13660                &module_ctx,
13661                health_response(health_frame.header.corr, HealthStatus::Ok),
13662            )
13663            .await
13664            .unwrap();
13665        let health_response = health_task.await.unwrap();
13666        assert_eq!(health_response[0].header.ty, FrameType::Response);
13667        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13668            ClientControlResponse::SupervisorHealthProbe {
13669                module_id, status, ..
13670            } => {
13671                assert_eq!(module_id, "mcp");
13672                assert_eq!(status, HealthStatus::Ok);
13673            }
13674            other => panic!("unexpected health response: {other:?}"),
13675        }
13676
13677        // Exercise the forwarding cleanup path directly while leaving the registry
13678        // advertisement in place. If cleanup leaves a stale control sink behind,
13679        // the next probe will enqueue onto it and wait for the long probe timeout
13680        // instead of returning an immediate no-connection error.
13681        forwarding
13682            .cleanup_connection(module_ctx.connection_id)
13683            .unwrap();
13684        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13685        let cleanup_response = tokio::time::timeout(
13686            Duration::from_millis(200),
13687            handler.handle_control_frame(
13688                &cleanup_probe_ctx,
13689                supervisor_health_probe_frame(303, "mcp"),
13690            ),
13691        )
13692        .await
13693        .expect("probe should fail immediately when the control lane is gone")
13694        .unwrap();
13695        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13696        assert_eq!(
13697            parse_error(&cleanup_response[0])["code"],
13698            "target_unavailable"
13699        );
13700        assert!(parse_error(&cleanup_response[0])["message"]
13701            .as_str()
13702            .unwrap()
13703            .contains("no module connection"));
13704
13705        handler
13706            .cleanup_connection(module_ctx.connection_id)
13707            .unwrap();
13708    }
13709
13710    #[tokio::test]
13711    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13712        let registry = Arc::new(Registry::default());
13713        let supervisor_handle = SupervisorHandle::new();
13714        let supervisor =
13715            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13716                .with_handle(supervisor_handle.clone())
13717                .with_connection_file_path(
13718                    std::env::temp_dir()
13719                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13720                );
13721        let module = supervisor
13722            .supervise_configured(
13723                ModuleSpec {
13724                    carry_containment: false,
13725                    module_id: "warming".to_string(),
13726                    program: fake_aft_stub_path(),
13727                    args: Vec::new(),
13728                    env: Vec::new(),
13729                    reserved: false,
13730                    reserved_prefixes: Vec::new(),
13731                    protocol: ModuleProtocol::Subc,
13732                    overlap: Default::default(),
13733                },
13734                true,
13735            )
13736            .unwrap();
13737        assert_eq!(module.state().unwrap(), ModuleState::Running);
13738
13739        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13740        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13741        let response = handler
13742            .handle_control_frame(
13743                &ctx,
13744                route_open_frame(304, "warming", unique_project_root("warming")),
13745            )
13746            .await
13747            .unwrap();
13748        module.stop().await.unwrap();
13749
13750        assert_eq!(response[0].header.ty, FrameType::Error);
13751        let error = parse_error(&response[0]);
13752        assert_eq!(error["code"], "module_warming");
13753        assert!(error["message"]
13754            .as_str()
13755            .unwrap()
13756            .contains("state=running, enabled=true, live=false"));
13757    }
13758
13759    #[test]
13760    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13761        let handler = ControlHandler::new(Arc::new(Registry::default()));
13762        let capture = EventCapture::default();
13763        let _subscriber =
13764            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13765        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13766        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13767        let pending = (0..limit).collect::<Vec<_>>();
13768        let response = handler
13769            .route_open_capacity_refusal(
13770                &ctx,
13771                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13772                "busy",
13773                pending.len(),
13774                limit,
13775            )
13776            .unwrap();
13777        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13778        let event = capture
13779            .events()
13780            .into_iter()
13781            .find(|event| {
13782                event.target == "control"
13783                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13784            })
13785            .expect("connection admission refusal event");
13786        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13787        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13788        // Refused before `handle_route_open` parses the request, so the caller
13789        // fields are read from the frame itself.
13790        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13791        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13792    }
13793
13794    #[test]
13795    fn route_open_target_cap_logs_admission_reason_and_capacity() {
13796        let handler = ControlHandler::new(Arc::new(Registry::default()));
13797        let capture = EventCapture::default();
13798        let _subscriber =
13799            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13800        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13801        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13802        let guards = (0..limit)
13803            .map(|_| {
13804                handler
13805                    .route_bind_concurrency
13806                    .try_admit("busy", limit)
13807                    .unwrap()
13808            })
13809            .collect::<Vec<_>>();
13810        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13811            Err(in_flight) => in_flight,
13812            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13813        };
13814        let response = handler
13815            .route_open_target_capacity_refusal(
13816                &ctx,
13817                &RouteOpenCaller::unknown(),
13818                &route_open_frame(397, "busy", unique_project_root("target-cap")),
13819                "busy",
13820                in_flight,
13821            )
13822            .unwrap();
13823        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13824        let event = capture
13825            .events()
13826            .into_iter()
13827            .find(|event| {
13828                event.target == "control"
13829                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13830            })
13831            .expect("target admission refusal event");
13832        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13833        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13834        drop(guards);
13835    }
13836
13837    #[cfg(unix)]
13838    #[tokio::test(start_paused = true)]
13839    async fn supervisor_list_shows_a_failed_modules_next_retry() {
13840        let registry = Arc::new(Registry::default());
13841        let handle = SupervisorHandle::new();
13842        let supervisor =
13843            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13844                .with_handle(handle.clone());
13845        let module = supervisor
13846            .spawn(crate::ModuleSpec {
13847                carry_containment: false,
13848                module_id: "retry-list".into(),
13849                program: PathBuf::from("/bin/sh"),
13850                args: vec!["-c".into(), "exit 17".into()],
13851                env: vec![],
13852                reserved: false,
13853                reserved_prefixes: vec![],
13854                protocol: ModuleProtocol::None,
13855                overlap: Default::default(),
13856            })
13857            .unwrap();
13858        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13859        while module.state().unwrap() != crate::ModuleState::Failed {
13860            assert!(
13861                std::time::Instant::now() < deadline,
13862                "waiting for budget failure"
13863            );
13864            tokio::task::yield_now().await;
13865        }
13866        let handler = ControlHandler::new(registry).with_supervisor(handle);
13867        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13868        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13869        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13870        assert_eq!(body["modules"][0]["state"], "failed");
13871        let retry = body["modules"][0]["next_retry_at_ms"]
13872            .as_u64()
13873            .expect("scheduled retry on the wire");
13874        let now = std::time::SystemTime::now()
13875            .duration_since(std::time::UNIX_EPOCH)
13876            .unwrap()
13877            .as_millis() as u64;
13878        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13879        module.set_enabled(false).await.unwrap();
13880        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13881        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13882        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13883        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13884    }
13885
13886    #[tokio::test]
13887    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
13888        let _guard = crate::subtree_resources::fault_read_guard().await;
13889        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
13890        let supervisor =
13891            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13892        let module = supervisor
13893            .spawn(crate::ModuleSpec {
13894                carry_containment: false,
13895                module_id: "pid-read-budget".into(),
13896                program: std::env::current_exe().unwrap(),
13897                args: vec![
13898                    "--exact".into(),
13899                    "control::tests::pid_resource_read_fixture".into(),
13900                ],
13901                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13902                    .into_iter()
13903                    .map(|key| (key.into(), dir.display().to_string()))
13904                    .chain(std::iter::once((
13905                        "SUBC_SUBTREE_PID_FIXTURE".into(),
13906                        "1".into(),
13907                    )))
13908                    .collect(),
13909                reserved: false,
13910                reserved_prefixes: Vec::new(),
13911                protocol: ModuleProtocol::None,
13912                overlap: Default::default(),
13913            })
13914            .unwrap();
13915        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
13916        while !matches!(
13917            module.child_resource_usage(),
13918            subc_control::ChildResourceUsage::Measured(_)
13919        ) {
13920            assert!(
13921                tokio::time::Instant::now() < ready_deadline,
13922                "fixture process was not measurable"
13923            );
13924            tokio::time::sleep(Duration::from_millis(10)).await;
13925        }
13926        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
13927        let reading = crate::subtree_resources::read_process(module.clone()).await;
13928        module.stop().await.unwrap();
13929        assert!(
13930            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
13931            "{reading:?}"
13932        );
13933    }
13934
13935    #[test]
13936    fn pid_resource_read_fixture() {
13937        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
13938            loop {
13939                std::thread::park_timeout(Duration::from_secs(600));
13940            }
13941        }
13942    }
13943
13944    #[tokio::test]
13945    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
13946        let _guard = crate::subtree_resources::fault_read_guard().await;
13947        let registry = Arc::new(Registry::default());
13948        let handle = SupervisorHandle::new();
13949        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13950            .with_handle(handle.clone());
13951        let module = supervisor
13952            .supervise_configured(
13953                crate::ModuleSpec {
13954                    carry_containment: false,
13955                    module_id: "memory-read".into(),
13956                    program: PathBuf::from("disabled-module"),
13957                    args: Vec::new(),
13958                    env: Vec::new(),
13959                    reserved: false,
13960                    reserved_prefixes: Vec::new(),
13961                    protocol: ModuleProtocol::Subc,
13962                    overlap: Default::default(),
13963                },
13964                false,
13965            )
13966            .unwrap();
13967        let (target, release, began) = crate::subtree_resources::held();
13968        module.set_subtree_target_for_test(target);
13969        let handler = ControlHandler::new(registry).with_supervisor(handle);
13970        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
13971        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13972        let replies = handler
13973            .handle_client_control_request(
13974                &ctx,
13975                frame,
13976                ClientControlRequest::SupervisorList {},
13977                &mut None,
13978            )
13979            .await
13980            .unwrap();
13981        assert!(
13982            replies.is_empty(),
13983            "list reply must be detached from the connection reader"
13984        );
13985        began.await.unwrap();
13986        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13987        let replies = handler
13988            .handle_client_control_request(
13989                &ctx,
13990                frame,
13991                ClientControlRequest::SupervisorSpawnSnapshot {},
13992                &mut None,
13993            )
13994            .await
13995            .unwrap();
13996        assert_eq!(
13997            replies.len(),
13998            1,
13999            "another request must complete while the subtree read stalls"
14000        );
14001        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14002            .await
14003            .unwrap()
14004            .unwrap();
14005        release.send(()).unwrap();
14006        drop(release);
14007        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14008        assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14009        assert_eq!(body["modules"][0]["subtree"]["reason"], "timeout");
14010        assert_eq!(
14011            body["modules"][0]["subtree"]["process_count"], 1,
14012            "a count obtained before the stalled read must survive timeout"
14013        );
14014        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14015        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14016        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14017        assert_eq!(
14018            body["modules"][0]["subtree"]["source"], "process_sum",
14019            "the next request must read afresh"
14020        );
14021    }
14022
14023    #[tokio::test]
14024    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14025        let registry = Arc::new(Registry::default());
14026        let handle = SupervisorHandle::new();
14027        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14028            .with_handle(handle.clone());
14029        supervisor
14030            .supervise_configured(
14031                crate::ModuleSpec {
14032                    carry_containment: false,
14033                    module_id: "drift".into(),
14034                    program: PathBuf::from("disabled-module"),
14035                    args: Vec::new(),
14036                    env: Vec::new(),
14037                    reserved: false,
14038                    reserved_prefixes: Vec::new(),
14039                    protocol: ModuleProtocol::Subc,
14040                    overlap: Default::default(),
14041                },
14042                false,
14043            )
14044            .unwrap();
14045        let handler = ControlHandler::new(registry).with_supervisor(handle);
14046        for configured in [None, Some(false), Some(true)] {
14047            handler.capability_evaluator.configure(
14048                configured.map(|enabled| ("drift".to_string(), enabled)),
14049                BTreeMap::new(),
14050            );
14051            let frame =
14052                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14053            let replies = handler
14054                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14055                .await
14056                .unwrap();
14057            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14058            assert!(body.get("configured_enabled").is_none());
14059            assert_eq!(body["op"], "supervisor.ack");
14060            let frame =
14061                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14062            let replies = handler.handle_supervisor_list(frame).await.unwrap();
14063            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14064            assert_eq!(
14065                body["modules"][0].get("configured_enabled"),
14066                configured.map(Value::Bool).as_ref()
14067            );
14068            assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14069            assert_eq!(body["modules"][0]["subtree"]["reason"], "not_running");
14070        }
14071    }
14072
14073    #[tokio::test(flavor = "current_thread")]
14074    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14075        let handler = ControlHandler::new(Arc::new(Registry::default()));
14076        let capture = EventCapture::default();
14077        let _subscriber =
14078            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14079        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14080        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14081
14082        let _ = handler
14083            .handle_client_control_request(
14084                &ctx,
14085                frame,
14086                ClientControlRequest::SupervisorSetEnabled {
14087                    module_id: "broca".to_string(),
14088                    enabled: false,
14089                },
14090                &mut None,
14091            )
14092            .await
14093            .unwrap();
14094
14095        let events = capture
14096            .events()
14097            .into_iter()
14098            .filter(|event| {
14099                event.target == "control"
14100                    && event.fields.get("message").map(String::as_str)
14101                        == Some("supervisor request received")
14102            })
14103            .collect::<Vec<_>>();
14104        assert_eq!(events.len(), 1, "one supervisor request log line");
14105        let event = &events[0];
14106        assert_eq!(event.level, tracing::Level::INFO);
14107        assert_eq!(
14108            event.fields.get("op"),
14109            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14110        );
14111        assert_eq!(
14112            event.fields.get("module_id"),
14113            Some(&"\"broca\"".to_string())
14114        );
14115        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14116        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14117        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14118    }
14119
14120    #[tokio::test(flavor = "current_thread")]
14121    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14122        let registry = Arc::new(Registry::default());
14123        let module_connection = ConnectionId::new(303);
14124        registry
14125            .register_with_control_ops(
14126                manifest("aft", PROTOCOL_VERSION),
14127                PROTOCOL_VERSION,
14128                module_connection,
14129                module_baseline_control_ops(),
14130            )
14131            .unwrap();
14132        let handler = ControlHandler::new(registry);
14133        let capture = EventCapture::default();
14134        let _subscriber =
14135            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14136        let (ctx, _rx) = route_ctx(module_connection);
14137        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14138
14139        let _ = handler
14140            .handle_client_control_request(
14141                &ctx,
14142                frame,
14143                ClientControlRequest::SupervisorSetEnabled {
14144                    module_id: "broca".to_string(),
14145                    enabled: false,
14146                },
14147                &mut None,
14148            )
14149            .await
14150            .unwrap();
14151
14152        let callers = capture
14153            .events()
14154            .into_iter()
14155            .filter(|event| {
14156                event.fields.get("message").map(String::as_str)
14157                    == Some("supervisor request received")
14158            })
14159            .map(|event| event.fields.get("caller").cloned())
14160            .collect::<Vec<_>>();
14161        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14162    }
14163
14164    #[tokio::test(flavor = "current_thread")]
14165    async fn supervisor_list_does_not_log_request_received() {
14166        let handler = ControlHandler::new(Arc::new(Registry::default()));
14167        let capture = EventCapture::default();
14168        let _subscriber =
14169            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14170        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14171        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14172
14173        handler
14174            .handle_client_control_request(
14175                &ctx,
14176                frame,
14177                ClientControlRequest::SupervisorList {},
14178                &mut None,
14179            )
14180            .await
14181            .unwrap();
14182
14183        assert!(capture.events().into_iter().all(|event| {
14184            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14185        }));
14186    }
14187
14188    #[tokio::test(flavor = "current_thread")]
14189    async fn supervisor_rescan_preview_does_not_log_request_received() {
14190        let handler = ControlHandler::new(Arc::new(Registry::default()));
14191        let capture = EventCapture::default();
14192        let _subscriber =
14193            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14194        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14195        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14196
14197        handler
14198            .handle_client_control_request(
14199                &ctx,
14200                frame,
14201                ClientControlRequest::SupervisorRescan { preview: true },
14202                &mut None,
14203            )
14204            .await
14205            .unwrap();
14206
14207        assert!(capture.events().into_iter().all(|event| {
14208            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14209        }));
14210    }
14211
14212    struct ApproveOperator;
14213    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14214        fn prompt(
14215            &self,
14216            _: &str,
14217            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14218        ) -> crate::operator_confirm::ProviderResult {
14219            crate::operator_confirm::ProviderResult::Approved
14220        }
14221    }
14222
14223    #[tokio::test]
14224    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14225        assert_eq!(
14226            ControlHandler::default().privileged_operator_prompt,
14227            PrivilegedOperatorPrompt::Log
14228        );
14229        for (initial, changed, running) in [
14230            ("log", "enforce", PrivilegedOperatorPrompt::Log),
14231            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14232        ] {
14233            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14234            let path = dir.join("subc.jsonc");
14235            std::fs::write(
14236                &path,
14237                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14238            )
14239            .unwrap();
14240            let config = crate::daemon_config::load(&path).unwrap().unwrap();
14241            let registry = Arc::new(Registry::default());
14242            let handle = SupervisorHandle::new();
14243            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14244                .with_handle(handle.clone());
14245            let handler = ControlHandler::new(registry)
14246                .with_supervisor(handle)
14247                .with_privileged_operator_prompt(config.privileged_operator_prompt)
14248                .with_supervisor_rescan(supervisor, &path, config.port);
14249            handler.forwarding.operator_confirms().configure(
14250                Arc::new(ApproveOperator),
14251                crate::operator_confirm::OperatorLimits::default(),
14252            );
14253            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14254
14255            for (disk_mode, needs_restart) in [
14256                (Some(initial), false),
14257                (Some(changed), true),
14258                (Some(changed), true),
14259                (None, running == PrivilegedOperatorPrompt::Enforce),
14260                (Some(initial), false),
14261            ] {
14262                let mut doc = json!({"version":1});
14263                if let Some(mode) = disk_mode {
14264                    doc["privileged_operator_prompt"] = json!(mode);
14265                }
14266                std::fs::write(&path, doc.to_string()).unwrap();
14267                for preview in [true, false] {
14268                    let request = ClientControlRequest::SupervisorRescan { preview };
14269                    let frame = Frame::build(
14270                        FrameType::Request,
14271                        control_flags(),
14272                        0,
14273                        0,
14274                        1,
14275                        serde_json::to_vec(&request).unwrap(),
14276                    )
14277                    .unwrap();
14278                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14279                    assert_eq!(frames.len(), 1);
14280                    let ClientControlResponse::SupervisorRescan { result } =
14281                        serde_json::from_slice(&frames[0].body).unwrap()
14282                    else {
14283                        panic!("expected a rescan response");
14284                    };
14285                    let expected = if needs_restart {
14286                        vec!["privileged_operator_prompt"]
14287                    } else {
14288                        Vec::new()
14289                    };
14290                    assert_eq!(result.restart_required, expected);
14291                    assert_eq!(handler.privileged_operator_prompt, running);
14292                    assert_eq!(
14293                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14294                        running
14295                    );
14296                }
14297            }
14298        }
14299    }
14300
14301    #[tokio::test]
14302    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14303        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14304        let path = dir.join("subc.jsonc");
14305        let registry = Arc::new(Registry::default());
14306        let handle = SupervisorHandle::new();
14307        let supervisor =
14308            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14309        let handler = ControlHandler::new(registry)
14310            .with_supervisor(handle)
14311            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14312            .with_supervisor_rescan(supervisor, &path, None);
14313        handler.forwarding.operator_confirms().configure(
14314            Arc::new(ApproveOperator),
14315            crate::operator_confirm::OperatorLimits::default(),
14316        );
14317        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14318        for preview in [true, false] {
14319            std::fs::write(
14320                &path,
14321                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14322            )
14323            .unwrap();
14324            let request = ClientControlRequest::SupervisorRescan { preview };
14325            let frame = Frame::build(
14326                FrameType::Request,
14327                control_flags(),
14328                0,
14329                0,
14330                1,
14331                serde_json::to_vec(&request).unwrap(),
14332            )
14333            .unwrap();
14334            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14335            assert_eq!(frames.len(), 1);
14336            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14337            assert_eq!(
14338                handler.privileged_operator_prompt,
14339                PrivilegedOperatorPrompt::Enforce
14340            );
14341        }
14342    }
14343
14344    /// One wire code has several senders, so the refusal line names the check
14345    /// that refused. This drives the shared refusal path for ordinary refusals
14346    /// with an unregistered
14347    /// target and requires the branch label on the event.
14348    #[tokio::test]
14349    async fn route_open_refusal_names_the_check_that_refused() {
14350        let handler = ControlHandler::new(Arc::new(Registry::default()));
14351        let capture = EventCapture::default();
14352        let _subscriber =
14353            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14354        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14355        let response = handler
14356            .handle_control_frame(
14357                &ctx,
14358                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14359            )
14360            .await
14361            .unwrap();
14362
14363        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14364        let event = capture
14365            .events()
14366            .into_iter()
14367            .find(|event| {
14368                event.target == "control"
14369                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14370            })
14371            .expect("route.open refusal event");
14372        assert_eq!(
14373            event.fields.get("reason"),
14374            Some(&"\"not_registered\"".to_string())
14375        );
14376    }
14377
14378    #[tokio::test]
14379    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14380        let registry = Arc::new(Registry::default());
14381        let supervisor_handle = SupervisorHandle::new();
14382        let supervisor =
14383            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14384                .with_handle(supervisor_handle.clone())
14385                .with_connection_file_path(std::env::temp_dir().join(format!(
14386                    "subc-route-open-refusal-info-{}",
14387                    std::process::id()
14388                )));
14389        let module = supervisor
14390            .supervise_configured(
14391                ModuleSpec {
14392                    carry_containment: false,
14393                    module_id: "warming".to_string(),
14394                    program: fake_aft_stub_path(),
14395                    args: Vec::new(),
14396                    env: Vec::new(),
14397                    reserved: false,
14398                    reserved_prefixes: Vec::new(),
14399                    protocol: ModuleProtocol::Subc,
14400                    overlap: Default::default(),
14401                },
14402                true,
14403            )
14404            .unwrap();
14405        assert_eq!(module.state().unwrap(), ModuleState::Running);
14406
14407        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14408        assert!(handler
14409            .counters()
14410            .snapshot()
14411            .get("route_open_refused_by_code")
14412            .is_none());
14413        let capture = EventCapture::default();
14414        let _subscriber =
14415            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14416        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14417        let response = handler
14418            .handle_control_frame(
14419                &ctx,
14420                route_open_frame(394, "warming", unique_project_root("refusal-info")),
14421            )
14422            .await
14423            .unwrap();
14424        module.stop().await.unwrap();
14425
14426        assert_eq!(parse_error(&response[0])["code"], "module_warming");
14427        let event = capture
14428            .events()
14429            .into_iter()
14430            .find(|event| {
14431                event.target == "control"
14432                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14433            })
14434            .expect("route.open refusal event");
14435        assert_eq!(
14436            event.fields.get("module_id"),
14437            Some(&"\"warming\"".to_string())
14438        );
14439        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14440        assert_eq!(
14441            event.fields.get("reason"),
14442            Some(&"\"supervised_not_registered\"".to_string())
14443        );
14444        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14445        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14446        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14447        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14448        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14449        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14450        assert_eq!(
14451            handler.counters().snapshot()["route_open_refused_by_code"],
14452            json!({ "module_warming": 1 })
14453        );
14454    }
14455
14456    const OUTAGE_START: &str = "route.open refusing module: not serving";
14457    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14458
14459    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14460        capture
14461            .events()
14462            .into_iter()
14463            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14464            .collect()
14465    }
14466
14467    fn supervise_stub(
14468        registry: &Arc<Registry>,
14469        module_id: &str,
14470        enabled: bool,
14471    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14472        let supervisor_handle = SupervisorHandle::new();
14473        let supervisor =
14474            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14475                .with_handle(supervisor_handle.clone())
14476                .with_connection_file_path(std::env::temp_dir().join(format!(
14477                    "subc-route-outage-{module_id}-{}",
14478                    std::process::id()
14479                )));
14480        let module = supervisor
14481            .supervise_configured(
14482                ModuleSpec {
14483                    carry_containment: false,
14484                    module_id: module_id.to_string(),
14485                    program: fake_aft_stub_path(),
14486                    args: Vec::new(),
14487                    env: Vec::new(),
14488                    reserved: false,
14489                    reserved_prefixes: Vec::new(),
14490                    protocol: ModuleProtocol::Subc,
14491                    overlap: Default::default(),
14492                },
14493                enabled,
14494            )
14495            .unwrap();
14496        (supervisor_handle, module)
14497    }
14498
14499    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14500        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14501            module_id: module_id.to_string(),
14502            drain_timeout_ms: Some(50),
14503        })
14504        .unwrap();
14505        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14506    }
14507
14508    /// Two handlers built over one forwarding table must share one outage
14509    /// tracker; separate trackers would each log their own opening line for
14510    /// the same outage.
14511    #[test]
14512    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14513        let registry = Arc::new(Registry::default());
14514        let forwarding = Arc::new(ForwardingTable::default());
14515        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14516        let second = ControlHandler::with_forwarding(registry, forwarding);
14517        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14518    }
14519
14520    /// A client can name any module id it likes. Refusing an unknown one,
14521    /// however often, must not create outage state or outage lines, or the
14522    /// tracker would be a memory sink any client could fill.
14523    #[tokio::test(flavor = "current_thread")]
14524    async fn route_open_unknown_module_refusals_add_no_outage_state() {
14525        let handler = ControlHandler::new(Arc::new(Registry::default()));
14526        let capture = EventCapture::default();
14527        let _subscriber =
14528            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14529        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14530        for corr in 0..8 {
14531            let response = handler
14532                .handle_control_frame(
14533                    &ctx,
14534                    route_open_frame(
14535                        380 + corr,
14536                        &format!("nobody-{corr}"),
14537                        unique_project_root("outage-unknown"),
14538                    ),
14539                )
14540                .await
14541                .unwrap();
14542            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14543        }
14544
14545        assert_eq!(handler.route_outages.tracked_module_count(), 0);
14546        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14547        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14548    }
14549
14550    /// Drives the refusal path end to end: a supervised module that served
14551    /// before and stopped being registered with no instruction to stop is a
14552    /// WARN, and the same module refused after an operator `supervisor.restart`
14553    /// is an INFO.
14554    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14555    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14556        let registry = Arc::new(Registry::default());
14557        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
14558        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14559        let capture = EventCapture::default();
14560        let _subscriber =
14561            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14562        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14563        // The stub never registers, so pretend it served once: otherwise every
14564        // refusal would fall in its startup window.
14565        handler.route_outages.record_accepted("outage-restart");
14566
14567        let response = handler
14568            .handle_control_frame(
14569                &ctx,
14570                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14571            )
14572            .await
14573            .unwrap();
14574        assert_eq!(response[0].header.ty, FrameType::Error);
14575        let starts = outage_lines(&capture, OUTAGE_START);
14576        assert_eq!(starts.len(), 1, "{starts:?}");
14577        assert_eq!(starts[0].level, tracing::Level::WARN);
14578        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14579        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14580        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14581        handler.route_outages.record_accepted("outage-restart");
14582        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14583
14584        let restart = handler
14585            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14586            .await
14587            .unwrap();
14588        assert_eq!(
14589            restart[0].header.ty,
14590            FrameType::Response,
14591            "{:?}",
14592            parse_error(&restart[0])
14593        );
14594        handler
14595            .handle_control_frame(
14596                &ctx,
14597                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14598            )
14599            .await
14600            .unwrap();
14601        module.stop().await.unwrap();
14602
14603        let starts = outage_lines(&capture, OUTAGE_START);
14604        assert_eq!(starts.len(), 2, "{starts:?}");
14605        assert_eq!(starts[1].level, tracing::Level::INFO);
14606        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14607    }
14608
14609    /// A restart refused before it touched the module (here: the module is
14610    /// disabled) must clear its operator mark, so the next real outage is
14611    /// still reported as a warning.
14612    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14613    async fn failed_operator_restart_leaves_no_operator_mark() {
14614        let registry = Arc::new(Registry::default());
14615        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
14616        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14617        let capture = EventCapture::default();
14618        let _subscriber =
14619            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14620        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14621        handler.route_outages.record_accepted("outage-disabled");
14622
14623        let restart = handler
14624            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14625            .await
14626            .unwrap();
14627        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14628        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14629
14630        handler
14631            .handle_control_frame(
14632                &ctx,
14633                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14634            )
14635            .await
14636            .unwrap();
14637        let starts = outage_lines(&capture, OUTAGE_START);
14638        assert_eq!(starts.len(), 1, "{starts:?}");
14639        assert_eq!(starts[0].level, tracing::Level::WARN);
14640    }
14641
14642    #[tokio::test(flavor = "current_thread")]
14643    async fn route_open_unknown_module_escapes_target_module_id() {
14644        let handler = ControlHandler::new(Arc::new(Registry::default()));
14645        let capture = EventCapture::default();
14646        let _subscriber =
14647            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14648        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14649        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14650        let response = handler
14651            .handle_control_frame(
14652                &ctx,
14653                route_open_frame(
14654                    395,
14655                    hostile_module_id,
14656                    unique_project_root("hostile-target-module-id"),
14657                ),
14658            )
14659            .await
14660            .unwrap();
14661
14662        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14663        let event = capture
14664            .events()
14665            .into_iter()
14666            .find(|event| {
14667                event.target == "control"
14668                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14669            })
14670            .expect("route.open unknown-module refusal event");
14671        let logged = event.fields.get("module_id").expect("module_id field");
14672        assert!(!logged.bytes().any(|byte| byte < 0x20));
14673        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14674    }
14675
14676    /// A route.open refusal is the daemon's own answer, so its ERROR carries
14677    /// the daemon-origin bit on the frame the client receives.
14678    #[tokio::test(flavor = "current_thread")]
14679    async fn route_open_refusal_carries_daemon_origin() {
14680        let handler = ControlHandler::new(Arc::new(Registry::default()));
14681        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14682        let response = handler
14683            .handle_control_frame(
14684                &ctx,
14685                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14686            )
14687            .await
14688            .unwrap();
14689
14690        assert_eq!(response.len(), 1);
14691        assert_eq!(response[0].header.ty, FrameType::Error);
14692        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14693        assert!(
14694            response[0].header.flags.is_daemon_origin(),
14695            "route.open refusal must carry daemon origin: {:?}",
14696            response[0].header.flags
14697        );
14698    }
14699
14700    /// The drain `module_reloading` refusal is daemon-synthesized for an
14701    /// endpoint that is reloading, so it carries the daemon-origin bit on the
14702    /// frame the client receives.
14703    #[tokio::test(flavor = "current_thread")]
14704    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14705        let registry = Arc::new(Registry::default());
14706        let forwarding = Arc::new(ForwardingTable::default());
14707        let handler =
14708            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14709        let module_connection = ConnectionId::new(95);
14710        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14711        hello_via_sink(
14712            &handler,
14713            &module_ctx,
14714            &mut module_rx,
14715            hello_frame("aft", PROTOCOL_VERSION, 395),
14716        )
14717        .await;
14718        forwarding
14719            .begin_module_drain("aft", RouteCloseReason::Restart)
14720            .unwrap()
14721            .expect("the module endpoint drains");
14722
14723        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14724        let response = handler
14725            .handle_control_frame(
14726                &client_ctx,
14727                route_open_frame(396, "aft", unique_project_root("drain-origin")),
14728            )
14729            .await
14730            .unwrap();
14731
14732        assert_eq!(response.len(), 1);
14733        assert_eq!(response[0].header.ty, FrameType::Error);
14734        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14735        assert!(
14736            response[0].header.flags.is_daemon_origin(),
14737            "drain module_reloading must carry daemon origin: {:?}",
14738            response[0].header.flags
14739        );
14740    }
14741
14742    /// A module that refuses a bind with a daemon code still cannot mint a
14743    /// daemon refusal: the relayed `module_reloading` body reaches the client
14744    /// with the daemon-origin bit clear.
14745    #[tokio::test(flavor = "current_thread")]
14746    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14747        let registry = Arc::new(Registry::default());
14748        let forwarding = Arc::new(ForwardingTable::default());
14749        let handler =
14750            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14751        let module_connection = ConnectionId::new(95);
14752        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14753        hello_via_sink(
14754            &handler,
14755            &module_ctx,
14756            &mut module_rx,
14757            hello_frame("aft", PROTOCOL_VERSION, 395),
14758        )
14759        .await;
14760
14761        let client_connection = ConnectionId::new(96);
14762        let (client_ctx, _client_rx) = route_ctx(client_connection);
14763        let (route_task, bind) = relay_route_open(
14764            &handler,
14765            client_connection,
14766            &client_ctx.egress,
14767            &mut module_rx,
14768            396,
14769            "aft",
14770            "spoofed-reloading",
14771        )
14772        .await;
14773        // The module sets the origin bit on its own rejection too: neither
14774        // the bit nor the daemon code may survive the relay as daemon-owned.
14775        let rejection = Frame::build(
14776            FrameType::Error,
14777            crate::router::daemon_error_flags(),
14778            0,
14779            0,
14780            bind.header.corr,
14781            serde_json::to_vec(&ErrorBody::new(
14782                "module_reloading",
14783                "module refused route.bind",
14784            ))
14785            .unwrap(),
14786        )
14787        .unwrap();
14788        handler
14789            .handle_control_frame(&module_ctx, rejection)
14790            .await
14791            .unwrap();
14792
14793        let response = route_task.await.unwrap();
14794        assert_eq!(response.len(), 1);
14795        assert_eq!(response[0].header.ty, FrameType::Error);
14796        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14797        assert!(
14798            !response[0].header.flags.is_daemon_origin(),
14799            "a module's own rejection must not carry daemon origin: {:?}",
14800            response[0].header.flags
14801        );
14802    }
14803
14804    #[tokio::test(flavor = "current_thread")]
14805    async fn route_open_module_rejection_uses_daemon_counter_key() {
14806        let registry = Arc::new(Registry::default());
14807        let forwarding = Arc::new(ForwardingTable::default());
14808        let handler =
14809            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14810        let module_connection = ConnectionId::new(95);
14811        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14812        hello_via_sink(
14813            &handler,
14814            &module_ctx,
14815            &mut module_rx,
14816            hello_frame("aft", PROTOCOL_VERSION, 395),
14817        )
14818        .await;
14819
14820        let client_connection = ConnectionId::new(96);
14821        let (client_ctx, _client_rx) = route_ctx(client_connection);
14822        let capture = EventCapture::default();
14823        let _subscriber =
14824            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14825        let (route_task, bind) = relay_route_open(
14826            &handler,
14827            client_connection,
14828            &client_ctx.egress,
14829            &mut module_rx,
14830            396,
14831            "aft",
14832            "hostile-module-code",
14833        )
14834        .await;
14835        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14836        let rejection = Frame::build(
14837            FrameType::Error,
14838            control_flags(),
14839            0,
14840            0,
14841            bind.header.corr,
14842            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14843        )
14844        .unwrap();
14845        handler
14846            .handle_control_frame(&module_ctx, rejection)
14847            .await
14848            .unwrap();
14849
14850        let response = route_task.await.unwrap();
14851        assert_eq!(parse_error(&response[0])["code"], hostile_code);
14852        let counters = handler.counters().snapshot();
14853        assert_eq!(
14854            counters["route_open_refused_by_code"],
14855            json!({ "module_rejected": 1 })
14856        );
14857        assert!(counters["route_open_refused_by_code"]
14858            .get(hostile_code)
14859            .is_none());
14860
14861        let event = capture
14862            .events()
14863            .into_iter()
14864            .find(|event| {
14865                event.target == "control"
14866                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14867            })
14868            .expect("route.open module-rejection refusal event");
14869        let logged = event.fields.get("module_code").expect("module_code field");
14870        assert!(!logged.bytes().any(|byte| byte < 0x20));
14871        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14872    }
14873
14874    #[tokio::test]
14875    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14876        let registry = Arc::new(Registry::default());
14877        let supervisor_handle = SupervisorHandle::new();
14878        let missing_program = std::env::temp_dir().join(format!(
14879            "subc-route-open-missing-program-{}",
14880            std::process::id()
14881        ));
14882        let supervisor =
14883            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14884                .with_handle(supervisor_handle.clone());
14885        let module = supervisor
14886            .supervise_configured(
14887                ModuleSpec {
14888                    carry_containment: false,
14889                    module_id: "failed".to_string(),
14890                    program: missing_program,
14891                    args: Vec::new(),
14892                    env: Vec::new(),
14893                    reserved: false,
14894                    reserved_prefixes: Vec::new(),
14895                    protocol: ModuleProtocol::Subc,
14896                    overlap: Default::default(),
14897                },
14898                true,
14899            )
14900            .unwrap();
14901        assert_eq!(module.state().unwrap(), ModuleState::Failed);
14902
14903        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14904        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14905        let response = handler
14906            .handle_control_frame(
14907                &ctx,
14908                route_open_frame(305, "failed", unique_project_root("failed")),
14909            )
14910            .await
14911            .unwrap();
14912
14913        assert_eq!(response[0].header.ty, FrameType::Error);
14914        let error = parse_error(&response[0]);
14915        assert_eq!(error["code"], "target_unavailable");
14916        assert!(error["message"]
14917            .as_str()
14918            .unwrap()
14919            .contains("state=failed, enabled=true, live=false"));
14920    }
14921
14922    #[tokio::test]
14923    async fn route_open_role_mismatch_remains_target_unavailable() {
14924        let registry = Arc::new(Registry::default());
14925        let handler = ControlHandler::new(Arc::clone(&registry));
14926        handler
14927            .handle_control(
14928                ConnectionId::new(41),
14929                non_routable_hello_frame_with_control_ops("health-only", 306, None),
14930            )
14931            .unwrap();
14932
14933        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14934        let response = handler
14935            .handle_control_frame(
14936                &ctx,
14937                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14938            )
14939            .await
14940            .unwrap();
14941
14942        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14943        assert!(parse_error(&response[0])["message"]
14944            .as_str()
14945            .unwrap()
14946            .contains("does not provide the requested target"));
14947    }
14948
14949    #[tokio::test]
14950    async fn route_open_inactive_registration_remains_target_unavailable() {
14951        let registry = Arc::new(Registry::default());
14952        let handler = ControlHandler::new(Arc::clone(&registry));
14953        handler
14954            .handle_control(
14955                ConnectionId::new(43),
14956                hello_frame("inactive", PROTOCOL_VERSION, 308),
14957            )
14958            .unwrap();
14959        assert!(registry
14960            .set_module_state_for_test("inactive", ChannelState::Closed)
14961            .unwrap());
14962
14963        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14964        let response = handler
14965            .handle_control_frame(
14966                &ctx,
14967                route_open_frame(309, "inactive", unique_project_root("inactive")),
14968            )
14969            .await
14970            .unwrap();
14971
14972        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14973        assert!(parse_error(&response[0])["message"]
14974            .as_str()
14975            .unwrap()
14976            .contains("is not active"));
14977    }
14978
14979    #[tokio::test]
14980    async fn late_health_reply_is_recorded_through_the_module_response_path() {
14981        let registry = Arc::new(Registry::default());
14982        let forwarding = Arc::new(ForwardingTable::default());
14983        let supervisor_handle = SupervisorHandle::new();
14984        let supervisor =
14985            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
14986                .with_forwarding(Arc::clone(&forwarding))
14987                .with_handle(supervisor_handle.clone());
14988        let module = supervisor
14989            .supervise_configured(
14990                crate::ModuleSpec {
14991                    carry_containment: false,
14992                    module_id: "late-health-response".to_string(),
14993                    program: PathBuf::from("disabled-module"),
14994                    args: Vec::new(),
14995                    env: Vec::new(),
14996                    reserved: false,
14997                    reserved_prefixes: Vec::new(),
14998                    protocol: ModuleProtocol::Subc,
14999                    overlap: Default::default(),
15000                },
15001                false,
15002            )
15003            .unwrap();
15004        let handler =
15005            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15006                .with_supervisor(supervisor_handle);
15007        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15008        handler
15009            .handle_control_frame(
15010                &module_ctx,
15011                hello_frame_with_control_ops(
15012                    "late-health-response",
15013                    PROTOCOL_VERSION,
15014                    7,
15015                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15016                ),
15017            )
15018            .await
15019            .unwrap();
15020        let probe_started_at = Instant::now() - Duration::from_millis(80);
15021        let pending = forwarding
15022            .begin_health_probe_rpc_for(
15023                "late-health-response",
15024                MODULE_CONTROL_OP_HEALTH_CHECK,
15025                probe_started_at,
15026                Instant::now() - Duration::from_millis(1),
15027            )
15028            .unwrap();
15029        assert!(forwarding
15030            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15031            .unwrap());
15032
15033        let responses = handler
15034            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15035            .await
15036            .unwrap();
15037
15038        assert!(responses.is_empty());
15039        let health = module.status().unwrap().health;
15040        assert_eq!(health.late_answer_count, 1);
15041        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15042    }
15043
15044    #[tokio::test]
15045    async fn health_probe_timeout_and_module_death_are_typed() {
15046        let registry = Arc::new(Registry::default());
15047        let forwarding = Arc::new(ForwardingTable::default());
15048        let handler =
15049            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15050                .with_health_probe_timeout(Duration::from_millis(50));
15051        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15052        hello_via_sink(
15053            &handler,
15054            &module_ctx,
15055            &mut module_rx,
15056            hello_frame_with_control_ops(
15057                "aft",
15058                PROTOCOL_VERSION,
15059                7,
15060                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15061            ),
15062        )
15063        .await;
15064
15065        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15066        let responses = handler
15067            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15068            .await
15069            .unwrap();
15070        assert_eq!(responses[0].header.ty, FrameType::Error);
15071        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15072        let _ = module_rx.try_recv();
15073
15074        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15075        let health_handler = handler.clone();
15076        let death_task = tokio::spawn(async move {
15077            health_handler
15078                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15079                .await
15080                .unwrap()
15081        });
15082        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15083            .await
15084            .unwrap()
15085            .unwrap();
15086        handler
15087            .cleanup_connection(module_ctx.connection_id)
15088            .unwrap();
15089        let responses = death_task.await.unwrap();
15090        assert_eq!(responses[0].header.ty, FrameType::Error);
15091        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15092    }
15093
15094    #[test]
15095    fn hello_requires_exact_protocol_version() {
15096        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15097            let registry = Arc::new(Registry::default());
15098            let handler = ControlHandler::new(Arc::clone(&registry));
15099            let responses = handler
15100                .handle_control(
15101                    ConnectionId::new(connection),
15102                    hello_frame("aft", offered, 9),
15103                )
15104                .unwrap();
15105
15106            assert_eq!(responses.len(), 1);
15107            assert_eq!(responses[0].header.ty, FrameType::Error);
15108            let error = parse_error(&responses[0]);
15109            assert_eq!(error["code"], "version_unsupported");
15110            assert!(registry.get_module("aft").unwrap().is_none());
15111            assert_eq!(registry.active_registration_count().unwrap(), 0);
15112        }
15113    }
15114
15115    #[test]
15116    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15117        let registry = Arc::new(Registry::default());
15118        let forwarding = Arc::new(ForwardingTable::default());
15119        let handler =
15120            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15121        let module_connection = ConnectionId::new(301);
15122        let registration = registry
15123            .register_with_control_ops(
15124                manifest("aft-push", PROTOCOL_VERSION),
15125                PROTOCOL_VERSION,
15126                module_connection,
15127                module_baseline_control_ops(),
15128            )
15129            .unwrap();
15130        let (module_tx, _module_rx) = mpsc::channel(8);
15131        let endpoint = forwarding
15132            .register_module_connection(
15133                module_connection,
15134                "aft-push".to_string(),
15135                PROTOCOL_VERSION,
15136                manifest_concurrency(&registration.manifest),
15137                FrameSink::new(module_tx),
15138            )
15139            .unwrap();
15140
15141        // A push op this version does not know is ignored (forward-compat), not errored.
15142        let unknown = Frame::build(
15143            FrameType::Push,
15144            control_flags(),
15145            0,
15146            0,
15147            5,
15148            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15149        )
15150        .unwrap();
15151        let out = handler.handle_status_update(endpoint, unknown).unwrap();
15152        assert!(
15153            out.is_empty(),
15154            "unknown push op must be ignored, got {out:?}"
15155        );
15156
15157        // A malformed body for a KNOWN op is a real error worth surfacing.
15158        let malformed = Frame::build(
15159            FrameType::Push,
15160            control_flags(),
15161            0,
15162            0,
15163            6,
15164            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15165        )
15166        .unwrap();
15167        let out = handler.handle_status_update(endpoint, malformed).unwrap();
15168        assert_eq!(out.len(), 1);
15169        assert_eq!(out[0].header.ty, FrameType::Error);
15170        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15171    }
15172
15173    #[test]
15174    fn hello_rejected_when_connection_already_owns_client_routes() {
15175        let registry = Arc::new(Registry::default());
15176        let forwarding = Arc::new(ForwardingTable::default());
15177        let handler =
15178            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15179        // Commits a client route on connection 202 (bound to a module on conn 101).
15180        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
15181        let client_connection = ConnectionId::new(202);
15182
15183        // That same connection now tries to register as a module: rejected, so one
15184        // connection never holds both client-route and module-endpoint state.
15185        let responses = handler
15186            .handle_control(
15187                client_connection,
15188                hello_frame("aft-second", PROTOCOL_VERSION, 9),
15189            )
15190            .unwrap();
15191        assert_eq!(responses[0].header.ty, FrameType::Error);
15192        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15193        assert!(registry.get_module("aft-second").unwrap().is_none());
15194    }
15195
15196    #[tokio::test]
15197    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15198        let registry = Arc::new(Registry::default());
15199        let forwarding = Arc::new(ForwardingTable::default());
15200        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15201        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15202        hello_via_sink(
15203            &handler,
15204            &module_ctx,
15205            &mut module_rx,
15206            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15207        )
15208        .await;
15209        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15210        let pending = forwarding
15211            .begin_route_bind_relay_for_test(
15212                client_ctx.connection_id,
15213                client_ctx.egress.clone(),
15214                2,
15215                "alpha",
15216            )
15217            .unwrap();
15218        forwarding
15219            .complete_pending_relay(
15220                module_ctx.connection_id,
15221                pending.corr,
15222                RouteBindRelayOutcome::Accepted,
15223            )
15224            .unwrap();
15225        client_rx.try_recv().unwrap();
15226        for module_id in ["beta", "alpha"] {
15227            let replies = handler
15228                .handle_control_frame(
15229                    &module_ctx,
15230                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15231                )
15232                .await
15233                .unwrap();
15234            assert_eq!(replies.len(), 1, "second HELLO must be refused");
15235            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15236        }
15237        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15238        assert!(registry.get_module("beta").unwrap().is_none());
15239        assert!(matches!(
15240            forwarding
15241                .lookup_data_route(
15242                    client_ctx.connection_id,
15243                    pending.client_channel,
15244                    pending.client_epoch,
15245                )
15246                .unwrap(),
15247            DataRoute::Client(DataRouteState::Bound(_))
15248        ));
15249        assert!(handler
15250            .hello_launch_nonces
15251            .lock()
15252            .unwrap()
15253            .presented(module_ctx.connection_id, Some("alpha-nonce")));
15254        assert!(module_rx.try_recv().is_err());
15255    }
15256
15257    #[test]
15258    fn reserved_module_hello_requires_matching_launch_nonce() {
15259        let registry = Arc::new(Registry::default());
15260        let supervisor = SupervisorHandle::new();
15261        // The supervisor recorded the nonce it injected when it spawned the reserved
15262        // module; the HELLO verifier checks against the same shared handle.
15263        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15264        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15265
15266        // A HELLO with NO nonce is rejected.
15267        let no_nonce = handler
15268            .handle_control(
15269                ConnectionId::new(1),
15270                hello_frame("vault", PROTOCOL_VERSION, 1),
15271            )
15272            .unwrap();
15273        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15274        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15275        assert!(registry.get_module("vault").unwrap().is_none());
15276
15277        // A HELLO with the WRONG nonce is rejected.
15278        let wrong = handler
15279            .handle_control(
15280                ConnectionId::new(2),
15281                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15282            )
15283            .unwrap();
15284        assert_eq!(wrong[0].header.ty, FrameType::Error);
15285        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15286        assert!(registry.get_module("vault").unwrap().is_none());
15287
15288        // A HELLO with the CORRECT nonce registers.
15289        let ok = handler
15290            .handle_control(
15291                ConnectionId::new(3),
15292                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15293            )
15294            .unwrap();
15295        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15296        assert!(registry.get_module("vault").unwrap().is_some());
15297    }
15298
15299    #[test]
15300    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15301        let registry = Arc::new(Registry::default());
15302        let supervisor = SupervisorHandle::new();
15303        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15304        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15305        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15306
15307        let squat = handler
15308            .handle_control(
15309                ConnectionId::new(1),
15310                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15311            )
15312            .unwrap();
15313        assert_eq!(squat[0].header.ty, FrameType::Error);
15314        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15315        assert!(parse_error(&squat[0])["message"]
15316            .as_str()
15317            .unwrap()
15318            .contains("fed:"));
15319
15320        let accepted_peer = handler
15321            .handle_control(
15322                ConnectionId::new(2),
15323                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15324            )
15325            .unwrap();
15326        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15327
15328        let accepted_short = handler
15329            .handle_control(
15330                ConnectionId::new(3),
15331                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15332            )
15333            .unwrap();
15334        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15335
15336        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15337            let response = handler
15338                .handle_control(
15339                    ConnectionId::new(conn),
15340                    hello_frame(module_id, PROTOCOL_VERSION, conn),
15341                )
15342                .unwrap();
15343            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15344        }
15345    }
15346
15347    #[test]
15348    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15349        let registry = Arc::new(Registry::default());
15350        let supervisor = SupervisorHandle::new();
15351        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15352        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15353        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15354        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15355
15356        let owner_nonce = handler
15357            .handle_control(
15358                ConnectionId::new(1),
15359                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15360            )
15361            .unwrap();
15362        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15363        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15364        assert!(registry.get_module("fed:special").unwrap().is_none());
15365
15366        let exact_nonce = handler
15367            .handle_control(
15368                ConnectionId::new(2),
15369                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15370            )
15371            .unwrap();
15372        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15373        assert!(registry.get_module("fed:special").unwrap().is_some());
15374    }
15375
15376    #[test]
15377    fn non_reserved_module_ignores_launch_nonce() {
15378        let registry = Arc::new(Registry::default());
15379        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
15380        // registration succeeds whether a spawned process echoes a nonce or not.
15381        let handler = ControlHandler::new(Arc::clone(&registry));
15382        let no_nonce = handler
15383            .handle_control(
15384                ConnectionId::new(1),
15385                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15386            )
15387            .unwrap();
15388        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15389        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15390
15391        let echoed_nonce = handler
15392            .handle_control(
15393                ConnectionId::new(2),
15394                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15395            )
15396            .unwrap();
15397        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15398        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15399    }
15400
15401    #[test]
15402    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15403        let handler = ControlHandler::default();
15404        let conn = ConnectionId::new(1);
15405        let malformed = Frame::build(
15406            FrameType::Hello,
15407            control_flags(),
15408            0,
15409            0,
15410            3,
15411            b"{not json".to_vec(),
15412        )
15413        .unwrap();
15414
15415        let error = handler.handle_control(conn, malformed).unwrap();
15416        assert_eq!(error[0].header.ty, FrameType::Error);
15417        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15418
15419        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15420        let pong = handler.handle_control(conn, ping).unwrap();
15421        assert_eq!(pong[0].header.ty, FrameType::Pong);
15422        assert_eq!(pong[0].header.corr, 4);
15423    }
15424
15425    #[test]
15426    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15427        let registry = Arc::new(Registry::default());
15428        let handler = ControlHandler::new(Arc::clone(&registry));
15429
15430        handler
15431            .handle_control(
15432                ConnectionId::new(1),
15433                hello_frame("aft", PROTOCOL_VERSION, 1),
15434            )
15435            .unwrap();
15436        let duplicate = handler
15437            .handle_control(
15438                ConnectionId::new(2),
15439                hello_frame("aft", PROTOCOL_VERSION, 2),
15440            )
15441            .unwrap();
15442
15443        assert_eq!(duplicate[0].header.ty, FrameType::Error);
15444        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15445        let registration = registry.get_module("aft").unwrap().unwrap();
15446        assert_eq!(registration.connection_id, ConnectionId::new(1));
15447    }
15448
15449    #[test]
15450    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15451        let registry = Arc::new(Registry::default());
15452        let forwarding = Arc::new(ForwardingTable::default());
15453        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15454        let handler =
15455            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15456                .with_process_liveness(process_liveness);
15457        let (ctx, route_channel, route_epoch) =
15458            bind_liveness_route(&registry, &forwarding, "aft-dead");
15459        let responses = handler
15460            .handle_route_poll(
15461                &ctx,
15462                route_poll_frame(41, PollKind::Liveness, route_channel),
15463                route_channel,
15464                route_epoch,
15465                PollKind::Liveness,
15466            )
15467            .unwrap();
15468
15469        assert_eq!(responses.len(), 1);
15470        assert_eq!(responses[0].header.ty, FrameType::Response);
15471        assert_route_poll_liveness(&responses[0], false);
15472    }
15473
15474    #[test]
15475    fn liveness_poll_without_process_source_uses_bound_route() {
15476        let registry = Arc::new(Registry::default());
15477        let forwarding = Arc::new(ForwardingTable::default());
15478        let handler =
15479            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15480        let (ctx, route_channel, route_epoch) =
15481            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
15482        let responses = handler
15483            .handle_route_poll(
15484                &ctx,
15485                route_poll_frame(42, PollKind::Liveness, route_channel),
15486                route_channel,
15487                route_epoch,
15488                PollKind::Liveness,
15489            )
15490            .unwrap();
15491
15492        assert_route_poll_liveness(&responses[0], true);
15493    }
15494
15495    #[test]
15496    fn liveness_poll_untracked_process_source_uses_bound_route() {
15497        let registry = Arc::new(Registry::default());
15498        let forwarding = Arc::new(ForwardingTable::default());
15499        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15500        let handler =
15501            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15502                .with_process_liveness(process_liveness);
15503        let (ctx, route_channel, route_epoch) =
15504            bind_liveness_route(&registry, &forwarding, "aft-untracked");
15505        let responses = handler
15506            .handle_route_poll(
15507                &ctx,
15508                route_poll_frame(43, PollKind::Liveness, route_channel),
15509                route_channel,
15510                route_epoch,
15511                PollKind::Liveness,
15512            )
15513            .unwrap();
15514
15515        assert_route_poll_liveness(&responses[0], true);
15516    }
15517
15518    #[tokio::test]
15519    async fn unknown_op_returns_unknown_control_op() {
15520        let handler = ControlHandler::default();
15521        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15522        let request = Frame::build(
15523            FrameType::Request,
15524            control_flags(),
15525            0,
15526            0,
15527            55,
15528            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15529        )
15530        .unwrap();
15531
15532        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15533
15534        assert_eq!(response.len(), 1);
15535        assert_eq!(response[0].header.ty, FrameType::Error);
15536        assert_eq!(response[0].header.corr, 55);
15537        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15538    }
15539
15540    #[tokio::test]
15541    async fn supervisor_provenance_rejects_unknown_exact_module() {
15542        let handler = ControlHandler::default();
15543        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15544        let request = Frame::build(
15545            FrameType::Request,
15546            control_flags(),
15547            0,
15548            0,
15549            57,
15550            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15551        )
15552        .unwrap();
15553
15554        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15555
15556        assert_eq!(response.len(), 1);
15557        assert_eq!(response[0].header.ty, FrameType::Error);
15558        assert_eq!(response[0].header.corr, 57);
15559        let error = parse_error(&response[0]);
15560        assert_eq!(error["code"], "unknown_module");
15561        assert_eq!(error["message"], "module_id 'missing' is not supervised");
15562    }
15563
15564    #[test]
15565    fn provenance_probe_override_keeps_handler_tests_deterministic() {
15566        let expected = subc_control::RunningImageAgreement::Unavailable {
15567            reason: subc_control::RunningImageUnavailableReason::HashFailed,
15568        };
15569        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15570        assert_eq!(handler.provenance_probe_override, Some(expected));
15571    }
15572
15573    #[test]
15574    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15575        let verdict = reload_verdict(
15576            std::path::Path::new("/bin/new"),
15577            Some(std::path::Path::new("/bin/old")),
15578            subc_control::RunningImageAgreement::Unavailable {
15579                reason: subc_control::RunningImageUnavailableReason::HashFailed,
15580            },
15581        );
15582        assert!(matches!(
15583            verdict.path,
15584            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15585                if configured == std::path::Path::new("/bin/new")
15586                    && spawned_from == std::path::Path::new("/bin/old")
15587        ));
15588    }
15589
15590    #[test]
15591    fn launch_spec_without_running_digest_is_unavailable_not_match() {
15592        use subc_control::{
15593            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15594        };
15595        let configured = crate::supervise::ModuleSpec {
15596            carry_containment: false,
15597            module_id: "legacy".into(),
15598            program: "/module".into(),
15599            args: Vec::new(),
15600            env: Vec::new(),
15601            reserved: false,
15602            reserved_prefixes: Vec::new(),
15603            protocol: subc_control::ModuleProtocol::Subc,
15604            overlap: Default::default(),
15605        };
15606        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15607        assert_eq!(
15608            launch_spec_agreement(&configured, false, Some(&digest)),
15609            Agreement::Unavailable {
15610                reason: Reason::NotRunning
15611            }
15612        );
15613        assert_eq!(
15614            launch_spec_agreement(&configured, true, None),
15615            Agreement::Unavailable {
15616                reason: Reason::SpawnedSpecUnavailable
15617            }
15618        );
15619        assert_eq!(
15620            launch_spec_agreement(&configured, true, Some(&digest)),
15621            Agreement::Match
15622        );
15623    }
15624
15625    #[cfg(windows)]
15626    #[test]
15627    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15628        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15629        let configured = dir.join("module.exe");
15630        std::fs::write(&configured, b"executable path fixture").unwrap();
15631        let spawned = std::fs::canonicalize(&configured).unwrap();
15632        assert_ne!(
15633            configured, spawned,
15634            "fixture must exercise different path spellings"
15635        );
15636        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15637        let lower_case = configured.to_string_lossy().to_lowercase();
15638        for spelling in [
15639            &configured,
15640            Path::new(&forward_slashes),
15641            Path::new(&lower_case),
15642        ] {
15643            let verdict = reload_verdict(
15644                spelling,
15645                Some(&spawned),
15646                subc_control::RunningImageAgreement::Unavailable {
15647                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15648                },
15649            );
15650            assert_eq!(
15651                verdict.path,
15652                ReloadPathAgreement::Match,
15653                "configured {spelling:?}, spawned {spawned:?}"
15654            );
15655        }
15656    }
15657
15658    #[cfg(windows)]
15659    #[test]
15660    fn windows_reload_path_keeps_different_existing_program_pending() {
15661        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15662        let old = dir.join("old.exe");
15663        let new = dir.join("new.exe");
15664        std::fs::write(&old, b"same executable content").unwrap();
15665        std::fs::copy(&old, &new).unwrap();
15666        let spawned = std::fs::canonicalize(&old).unwrap();
15667        let image = subc_control::RunningImageAgreement::Unavailable {
15668            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15669        };
15670        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15671        assert!(
15672            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15673            if configured == new && spawned_from == spawned)
15674        );
15675        let missing = dir.join("missing.exe");
15676        assert!(matches!(
15677            reload_verdict(&missing, Some(&spawned), image).path,
15678            ReloadPathAgreement::Mismatch { .. }
15679        ));
15680    }
15681
15682    #[test]
15683    fn reload_verdict_detects_replaced_image_at_same_path() {
15684        let image = subc_control::RunningImageAgreement::Mismatch {
15685            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15686                digest: "old".into(),
15687            },
15688            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15689                digest: "new".into(),
15690            },
15691        };
15692        let verdict = reload_verdict(
15693            std::path::Path::new("/bin/same"),
15694            Some(std::path::Path::new("/bin/same")),
15695            image.clone(),
15696        );
15697        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15698        assert_eq!(verdict.image, image);
15699    }
15700
15701    #[test]
15702    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15703        let image = subc_control::RunningImageAgreement::Unavailable {
15704            reason: subc_control::RunningImageUnavailableReason::NotRunning,
15705        };
15706        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15707        assert_eq!(
15708            verdict.path,
15709            subc_control::ReloadPathAgreement::Unavailable {
15710                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15711            }
15712        );
15713        assert_eq!(verdict.image, image);
15714
15715        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15716            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15717        };
15718        let verdict = reload_verdict(
15719            std::path::Path::new("/bin/same"),
15720            Some(std::path::Path::new("/bin/same")),
15721            unconfirmed.clone(),
15722        );
15723        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15724        assert_eq!(verdict.image, unconfirmed);
15725    }
15726
15727    #[test]
15728    fn reload_verdict_preserves_each_image_unavailability_reason() {
15729        use subc_control::RunningImageUnavailableReason as Reason;
15730
15731        for reason in [
15732            Reason::NotRunning,
15733            Reason::UnsupportedPlatform,
15734            Reason::RunningExecutableUnreadable,
15735            Reason::SpawnedPathUnreadable,
15736            Reason::HashFailed,
15737            Reason::ProcessIdentityUnconfirmed,
15738            Reason::Unknown("future_probe_reason".to_string()),
15739        ] {
15740            let image = subc_control::RunningImageAgreement::Unavailable {
15741                reason: reason.clone(),
15742            };
15743            let verdict = reload_verdict(
15744                std::path::Path::new("/bin/same"),
15745                Some(std::path::Path::new("/bin/same")),
15746                image.clone(),
15747            );
15748            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15749            assert_eq!(verdict.image, image, "{reason:?}");
15750        }
15751    }
15752
15753    #[tokio::test]
15754    async fn malformed_control_bodies_return_invalid_control_body() {
15755        let handler = ControlHandler::default();
15756        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15757
15758        for (corr, body) in [
15759            (56, br#"{"route_channel":1}"#.as_slice()),
15760            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15761            (
15762                58,
15763                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15764            ),
15765        ] {
15766            let request = Frame::build(
15767                FrameType::Request,
15768                control_flags(),
15769                0,
15770                0,
15771                corr,
15772                body.to_vec(),
15773            )
15774            .unwrap();
15775            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15776
15777            assert_eq!(response.len(), 1);
15778            assert_eq!(response[0].header.ty, FrameType::Error);
15779            assert_eq!(response[0].header.corr, corr);
15780            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15781        }
15782    }
15783
15784    #[tokio::test]
15785    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15786        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15787        let registry = Arc::new(Registry::default());
15788        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15789        let router = Router::with_control_handler(Arc::clone(&control));
15790        let connection = router.begin_connection();
15791        let (ctx, mut rx) = route_ctx(connection.id());
15792
15793        router
15794            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15795            .await
15796            .unwrap();
15797        let response = rx.recv().await.unwrap();
15798        let ack = parse_ack(&response);
15799        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15800        let channel = 1;
15801
15802        let goodbye =
15803            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15804        router.route_for_connection(&ctx, goodbye).await.unwrap();
15805        assert!(rx.try_recv().is_err());
15806        assert!(registry.get_module("aft").unwrap().is_none());
15807
15808        router
15809            .route_for_connection(&ctx, channel_request(channel, 13))
15810            .await
15811            .unwrap();
15812        let error_frame = rx.recv().await.unwrap();
15813        assert_eq!(error_frame.header.ty, FrameType::Error);
15814        assert_eq!(error_frame.header.channel, channel);
15815        let captured = crate::router::test_log::captured_logs(&logs);
15816        assert_eq!(
15817            captured
15818                .lines()
15819                .filter(|line| {
15820                    line.contains("module_id=aft")
15821                        && line.contains("reason=explicit_goodbye")
15822                        && line.contains("module registration ended")
15823                })
15824                .count(),
15825            1,
15826            "unexpected GOODBYE registry log: {captured}"
15827        );
15828    }
15829
15830    #[tokio::test]
15831    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15832        let registry = Arc::new(Registry::default());
15833        let handler = ControlHandler::new(registry).with_capability_config(
15834            [("prov".to_string(), true), ("cons".to_string(), true)],
15835            BTreeMap::new(),
15836        );
15837        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15838        register_capability_manifest(
15839            &handler,
15840            &provider_ctx,
15841            &mut provider_rx,
15842            capability_manifest("prov", &["thing/v1"], &[]),
15843            1,
15844        )
15845        .await;
15846        let mut consumer = capability_manifest("cons", &[], &[]);
15847        consumer.capabilities.as_mut().unwrap().requires.push(
15848            subc_protocol::manifest::CapabilityRequirement {
15849                capability: "thing/v1".to_string(),
15850                need: subc_protocol::manifest::CapabilityNeed::Required,
15851            },
15852        );
15853        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15854        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15855        assert_eq!(
15856            handler.capability_evaluator.verdict("cons", "thing/v1"),
15857            Some(CapabilityVerdict::Provided)
15858        );
15859        let (mut client_rx, _) = open_route_for_capability_test(
15860            &handler,
15861            &provider_ctx,
15862            &mut provider_rx,
15863            703,
15864            3,
15865            "prov",
15866            None,
15867        )
15868        .await;
15869        let goodbye =
15870            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15871        handler
15872            .handle_control_frame(&provider_ctx, goodbye)
15873            .await
15874            .unwrap();
15875        assert_eq!(
15876            handler.capability_evaluator.verdict("cons", "thing/v1"),
15877            Some(CapabilityVerdict::NeverProvided)
15878        );
15879        let closed = client_rx
15880            .try_recv()
15881            .expect("GOODBYE pushes route.closed before route GOODBYE");
15882        assert!(
15883            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15884            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15885        );
15886        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15887        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15888    }
15889
15890    struct RouteClosedFixture {
15891        handler: ControlHandler,
15892        #[cfg(unix)]
15893        supervisor: Supervisor,
15894        module_ctx: RouteCtx,
15895        #[cfg(unix)]
15896        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15897        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15898        route: Arc<crate::forwarding::RouteBinding>,
15899    }
15900
15901    async fn route_closed_fixture() -> RouteClosedFixture {
15902        let registry = Arc::new(Registry::default());
15903        let handler = ControlHandler::new(Arc::clone(&registry));
15904        #[cfg(unix)]
15905        let supervisor = Supervisor::new(registry, RestartPolicy::default())
15906            .with_forwarding(Arc::clone(&handler.forwarding));
15907        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15908        register_capability_manifest(
15909            &handler,
15910            &module_ctx,
15911            &mut module_rx,
15912            capability_manifest("prov", &[], &[]),
15913            1,
15914        )
15915        .await;
15916        let (client_rx, bind) = open_route_for_capability_test(
15917            &handler,
15918            &module_ctx,
15919            &mut module_rx,
15920            702,
15921            2,
15922            "prov",
15923            None,
15924        )
15925        .await;
15926        let ModuleControlRequest::RouteBind {
15927            route_channel: channel,
15928            epoch: route_epoch,
15929            ..
15930        } = bind
15931        else {
15932            panic!("expected route.bind");
15933        };
15934        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15935            handler
15936                .forwarding
15937                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15938                .unwrap()
15939        else {
15940            panic!("the acknowledged route must be bound");
15941        };
15942        RouteClosedFixture {
15943            handler,
15944            #[cfg(unix)]
15945            supervisor,
15946            module_ctx,
15947            #[cfg(unix)]
15948            module_rx,
15949            client_rx,
15950            route,
15951        }
15952    }
15953
15954    impl RouteClosedFixture {
15955        #[cfg(unix)]
15956        async fn cut(&mut self) {
15957            let notices = async {
15958                let mut draining = self.module_rx.recv().await.unwrap();
15959                assert!(matches!(
15960                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15961                        &draining.body
15962                    )
15963                    .unwrap(),
15964                    subc_protocol::session::ModuleControlCommand::Draining {
15965                        reason: RouteCloseReason::Disable,
15966                        ..
15967                    }
15968                ));
15969                draining.flushed.take().unwrap().send(()).unwrap();
15970                let mut closing = self.client_rx.recv().await.unwrap();
15971                assert!(matches!(
15972                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15973                    ClientControlPush::RouteClosing {
15974                        reason: RouteCloseReason::Restart,
15975                        ..
15976                    }
15977                ));
15978                closing.flushed.take().unwrap().send(()).unwrap();
15979            };
15980            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15981            result.unwrap();
15982        }
15983
15984        fn disconnect(&mut self) -> ClientControlPush {
15985            self.handler
15986                .cleanup_connection(self.module_ctx.connection_id)
15987                .unwrap();
15988            let closed = self.client_rx.try_recv().unwrap();
15989            assert_eq!(closed.header.ty, FrameType::Push);
15990            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15991            assert!(matches!(
15992                &push,
15993                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15994                    if channels == &[self.route.client_channel]
15995            ));
15996            assert_eq!(
15997                self.client_rx.try_recv().unwrap().header.ty,
15998                FrameType::Goodbye
15999            );
16000            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16001            push
16002        }
16003    }
16004
16005    #[cfg(unix)]
16006    #[tokio::test(start_paused = true)]
16007    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16008        let mut fixture = route_closed_fixture().await;
16009        fixture.cut().await;
16010        assert!(matches!(
16011            fixture.disconnect(),
16012            ClientControlPush::RouteClosed {
16013                reason: RouteCloseReason::Restart,
16014                drained: true,
16015                ..
16016            }
16017        ));
16018    }
16019
16020    #[cfg(unix)]
16021    #[tokio::test(start_paused = true)]
16022    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16023        let mut fixture = route_closed_fixture().await;
16024        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16025        let started = tokio::time::Instant::now();
16026        fixture.cut().await;
16027        assert!(started.elapsed() >= Duration::from_secs(2));
16028        assert_eq!(fixture.route.flow.in_flight(), 1);
16029        assert!(matches!(
16030            fixture.disconnect(),
16031            ClientControlPush::RouteClosed {
16032                reason: RouteCloseReason::Restart,
16033                drained: false,
16034                ..
16035            }
16036        ));
16037    }
16038
16039    #[tokio::test]
16040    async fn module_connection_crash_reports_crash_and_not_drained() {
16041        let mut fixture = route_closed_fixture().await;
16042        // The server calls cleanup_connection on module EOF. Without a planned
16043        // daemon shutdown, that disconnect must remain a crash with no prior
16044        // route.closing notice, even when the route has no in-flight calls.
16045        assert!(matches!(
16046            fixture.disconnect(),
16047            ClientControlPush::RouteClosed {
16048                reason: RouteCloseReason::Crash,
16049                drained: false,
16050                ..
16051            }
16052        ));
16053    }
16054
16055    #[tokio::test]
16056    async fn dropping_router_connection_releases_registration() {
16057        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16058        let registry = Arc::new(Registry::default());
16059        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16060        let router = Router::with_control_handler(Arc::clone(&control));
16061        let connection = router.begin_connection();
16062        let connection_id = connection.id();
16063        let (ctx, mut rx) = route_ctx(connection_id);
16064
16065        router
16066            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16067            .await
16068            .unwrap();
16069        let response = rx.recv().await.unwrap();
16070        let ack = parse_ack(&response);
16071        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16072        assert!(registry.get_module("aft").unwrap().is_some());
16073
16074        drop(connection);
16075
16076        assert!(registry.get_module("aft").unwrap().is_none());
16077        assert_eq!(registry.active_registration_count().unwrap(), 0);
16078
16079        control
16080            .cleanup_connection(ConnectionId::new(u64::MAX))
16081            .unwrap();
16082        let captured = crate::router::test_log::captured_logs(&logs);
16083        println!("captured registry lifecycle logs:\n{captured}");
16084        let events: Vec<_> = captured
16085            .lines()
16086            .filter(|line| {
16087                line.contains("module registered module_id=aft ")
16088                    || line.contains("module registration ended")
16089            })
16090            .collect();
16091        assert_eq!(
16092            events.len(),
16093            2,
16094            "unexpected registry lifecycle logs: {captured}"
16095        );
16096        assert!(events[0].contains("module registered module_id=aft "));
16097        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16098        assert!(events[1].contains(&format!(
16099            "module_id=aft connection_id={}",
16100            connection_id.get()
16101        )));
16102        assert!(events[1].contains("reason=connection_closed"));
16103        assert!(events[1].contains("module registration ended"));
16104        assert_eq!(
16105            captured
16106                .lines()
16107                .filter(|line| line.contains("module registered module_id=aft "))
16108                .count(),
16109            1,
16110            "legacy registration admission line must appear once: {captured}"
16111        );
16112    }
16113
16114    #[tokio::test]
16115    async fn hello_registration_keeps_legacy_module_registered_line_once() {
16116        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16117        let registry = Arc::new(Registry::default());
16118        let control = ControlHandler::new(Arc::clone(&registry));
16119        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16120        hello_via_sink(
16121            &control,
16122            &ctx,
16123            &mut rx,
16124            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16125        )
16126        .await;
16127
16128        let captured = crate::router::test_log::captured_logs(&logs);
16129        let admissions: Vec<_> = captured
16130            .lines()
16131            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16132            .collect();
16133        assert_eq!(
16134            admissions.len(),
16135            1,
16136            "legacy admission line must remain exactly once: {captured}"
16137        );
16138        assert!(admissions[0].contains("routable_provider=true"));
16139        assert!(admissions[0].contains("connection_id=777"));
16140    }
16141
16142    fn capability_manifest(
16143        module_id: &str,
16144        provides: &[&str],
16145        must_never_reach: &[&str],
16146    ) -> ModuleManifest {
16147        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16148        manifest.capabilities = Some(CapabilityDeclarations {
16149            provides: provides
16150                .iter()
16151                .map(|capability| (*capability).to_string())
16152                .collect(),
16153            requires: Vec::new(),
16154            must_never_reach: must_never_reach
16155                .iter()
16156                .map(|capability| (*capability).to_string())
16157                .collect(),
16158        });
16159        manifest
16160    }
16161
16162    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16163        Frame::build(
16164            FrameType::Hello,
16165            control_flags(),
16166            0,
16167            0,
16168            corr,
16169            serde_json::to_vec(&ModuleHelloBody {
16170                protocol_ver: manifest.protocol_ver,
16171                manifest,
16172                control_ops: None,
16173                launch_nonce: None,
16174            })
16175            .expect("capability test HELLO serializes"),
16176        )
16177        .expect("capability test HELLO frame builds")
16178    }
16179
16180    fn catalog_update_with_capabilities_frame(
16181        corr: u64,
16182        capabilities: CapabilityDeclarations,
16183    ) -> Frame {
16184        Frame::build(
16185            FrameType::Request,
16186            control_flags(),
16187            0,
16188            0,
16189            corr,
16190            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16191                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16192                capabilities: Some(capabilities),
16193                ready: None,
16194            })
16195            .expect("capability catalog.update serializes"),
16196        )
16197        .expect("capability catalog.update frame builds")
16198    }
16199
16200    async fn register_capability_manifest(
16201        handler: &ControlHandler,
16202        ctx: &RouteCtx,
16203        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16204        manifest: ModuleManifest,
16205        corr: u64,
16206    ) {
16207        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16208    }
16209
16210    async fn open_route_for_capability_test(
16211        handler: &ControlHandler,
16212        target_ctx: &RouteCtx,
16213        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16214        client_connection_id: u64,
16215        corr: u64,
16216        target_module_id: &str,
16217        consumer_identity: Option<ConsumerIdentity>,
16218    ) -> (
16219        mpsc::Receiver<crate::router::OutboundFrame>,
16220        ModuleControlRequest,
16221    ) {
16222        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16223        let route_handler = handler.clone();
16224        let target_module_id = target_module_id.to_string();
16225        let route_task = tokio::spawn(async move {
16226            route_handler
16227                .handle_control_frame(
16228                    &client_ctx,
16229                    route_open_frame_with_admission_facts(
16230                        corr,
16231                        &target_module_id,
16232                        unique_project_root("admission-facts"),
16233                        consumer_identity,
16234                        None,
16235                    ),
16236                )
16237                .await
16238                .expect("capability test route.open succeeds")
16239        });
16240        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16241            .await
16242            .expect("capability test route.open must reach route.bind")
16243            .expect("target control receiver stays open");
16244        let bind_request: ModuleControlRequest =
16245            serde_json::from_slice(&bind.body).expect("route.bind decodes");
16246        handler
16247            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16248            .await
16249            .expect("capability test route.bind ACK succeeds");
16250        assert!(route_task.await.expect("route.open task joins").is_empty());
16251        let opened = client_rx
16252            .recv()
16253            .await
16254            .expect("successful route.open publishes a response");
16255        assert!(matches!(
16256            serde_json::from_slice::<ClientControlResponse>(&opened.body),
16257            Ok(ClientControlResponse::RouteOpen { .. })
16258        ));
16259        (client_rx, bind_request)
16260    }
16261
16262    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16263        assert_eq!(frame.header.ty, FrameType::Push);
16264        assert_eq!(frame.header.channel, 0);
16265        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16266            .expect("route.closed control push decodes");
16267        let ClientControlPush::RouteClosed { channels, .. } = &push else {
16268            panic!("expected route.closed");
16269        };
16270        assert_eq!(channels.len(), 1, "exactly one violating route closed");
16271        let closed_channel = channels[0];
16272        let channels = channels.clone();
16273        assert_eq!(
16274            push,
16275            ClientControlPush::RouteClosed {
16276                module_id: target_module_id.to_string(),
16277                channels,
16278                reason: RouteCloseReason::CapabilityDenied,
16279                drained: false,
16280                abandoned: 0,
16281                excluded_subscriptions: 0,
16282                terminal: Some(false),
16283            }
16284        );
16285        closed_channel
16286    }
16287
16288    #[tokio::test]
16289    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16290        let registry = Arc::new(Registry::default());
16291        let forwarding = Arc::new(ForwardingTable::default());
16292        let supervisor = SupervisorHandle::new();
16293        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16294        let handler =
16295            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16296                .with_supervisor(supervisor);
16297        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16298        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16299        register_capability_manifest(
16300            &handler,
16301            &target_ctx,
16302            &mut target_rx,
16303            capability_manifest("target", &["credentials-provider/v1"], &[]),
16304            1,
16305        )
16306        .await;
16307        register_capability_manifest(
16308            &handler,
16309            &opener_ctx,
16310            &mut opener_rx,
16311            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16312            2,
16313        )
16314        .await;
16315
16316        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16317        let replies = handler
16318            .handle_control_frame(
16319                &client_ctx,
16320                route_open_frame_with_admission_facts(
16321                    3,
16322                    "target",
16323                    unique_project_root("admission-facts"),
16324                    Some(ConsumerIdentity {
16325                        module_id: "opener".to_string(),
16326                        launch_nonce: "opener-nonce".to_string(),
16327                    }),
16328                    None,
16329                ),
16330            )
16331            .await
16332            .expect("denied route.open returns a typed frame");
16333        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16334        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16335        assert!(
16336            target_rx.try_recv().is_err(),
16337            "forbidden route.open must not relay route.bind"
16338        );
16339    }
16340
16341    #[tokio::test]
16342    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16343        let registry = Arc::new(Registry::default());
16344        let forwarding = Arc::new(ForwardingTable::default());
16345        let supervisor = SupervisorHandle::new();
16346        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16347        let handler =
16348            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16349                .with_supervisor(supervisor);
16350        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16351        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16352        register_capability_manifest(
16353            &handler,
16354            &target_ctx,
16355            &mut target_rx,
16356            capability_manifest("target", &["credentials-provider/v1"], &[]),
16357            1,
16358        )
16359        .await;
16360        register_capability_manifest(
16361            &handler,
16362            &old_opener_ctx,
16363            &mut old_opener_rx,
16364            capability_manifest("opener", &[], &[]),
16365            2,
16366        )
16367        .await;
16368        let (mut client_rx, _) = open_route_for_capability_test(
16369            &handler,
16370            &target_ctx,
16371            &mut target_rx,
16372            712,
16373            3,
16374            "target",
16375            Some(ConsumerIdentity {
16376                module_id: "opener".to_string(),
16377                launch_nonce: "opener-nonce".to_string(),
16378            }),
16379        )
16380        .await;
16381        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16382
16383        handler
16384            .cleanup_connection(old_opener_ctx.connection_id)
16385            .expect("old opener registration cleans up");
16386        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16387        register_capability_manifest(
16388            &handler,
16389            &new_opener_ctx,
16390            &mut new_opener_rx,
16391            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16392            4,
16393        )
16394        .await;
16395
16396        let closed_channel = assert_capability_denied_push(
16397            client_rx
16398                .try_recv()
16399                .expect("HELLO deny addition must emit route.closed")
16400                .frame,
16401            "target",
16402        );
16403        let goodbye = client_rx
16404            .try_recv()
16405            .expect("HELLO deny addition must send the client a route GOODBYE")
16406            .frame;
16407        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16408        assert_eq!(goodbye.header.channel, closed_channel);
16409        assert_ne!(goodbye.header.channel, 0);
16410        assert_ne!(goodbye.header.epoch, 0);
16411        assert!(client_rx.try_recv().is_err());
16412        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16413        assert!(matches!(
16414            target_rx.try_recv(),
16415            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16416        ));
16417    }
16418
16419    #[tokio::test]
16420    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16421        let registry = Arc::new(Registry::default());
16422        let forwarding = Arc::new(ForwardingTable::default());
16423        let supervisor = SupervisorHandle::new();
16424        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16425        let handler =
16426            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16427                .with_supervisor(supervisor);
16428        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16429        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16430        register_capability_manifest(
16431            &handler,
16432            &target_ctx,
16433            &mut target_rx,
16434            capability_manifest("target", &[], &[]),
16435            1,
16436        )
16437        .await;
16438        register_capability_manifest(
16439            &handler,
16440            &opener_ctx,
16441            &mut opener_rx,
16442            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16443            2,
16444        )
16445        .await;
16446        let (mut client_rx, _) = open_route_for_capability_test(
16447            &handler,
16448            &target_ctx,
16449            &mut target_rx,
16450            722,
16451            3,
16452            "target",
16453            Some(ConsumerIdentity {
16454                module_id: "opener".to_string(),
16455                launch_nonce: "opener-nonce".to_string(),
16456            }),
16457        )
16458        .await;
16459        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16460
16461        let replies = handler
16462            .handle_control_frame(
16463                &target_ctx,
16464                catalog_update_with_capabilities_frame(
16465                    4,
16466                    CapabilityDeclarations {
16467                        provides: vec!["credentials-provider/v1".to_string()],
16468                        requires: Vec::new(),
16469                        must_never_reach: Vec::new(),
16470                    },
16471                ),
16472            )
16473            .await
16474            .expect("claim catalog.update succeeds");
16475        assert!(matches!(
16476            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16477            Ok(ModuleControlResponseToModule::CatalogUpdate {})
16478        ));
16479        let closed_channel = assert_capability_denied_push(
16480            client_rx
16481                .try_recv()
16482                .expect("claim addition must emit route.closed")
16483                .frame,
16484            "target",
16485        );
16486        let goodbye = client_rx
16487            .try_recv()
16488            .expect("claim addition must send the client a route GOODBYE")
16489            .frame;
16490        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16491        assert_eq!(goodbye.header.channel, closed_channel);
16492        assert_ne!(goodbye.header.channel, 0);
16493        assert_ne!(goodbye.header.epoch, 0);
16494        assert!(client_rx.try_recv().is_err());
16495        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16496        assert!(matches!(
16497            target_rx.try_recv(),
16498            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16499        ));
16500    }
16501
16502    #[tokio::test]
16503    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16504        let registry = Arc::new(Registry::default());
16505        let forwarding = Arc::new(ForwardingTable::default());
16506        let supervisor = SupervisorHandle::new();
16507        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16508        let handler =
16509            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16510                .with_supervisor(supervisor);
16511        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16512        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16513        register_capability_manifest(
16514            &handler,
16515            &target_ctx,
16516            &mut target_rx,
16517            capability_manifest("target", &["credentials-provider/v1"], &[]),
16518            1,
16519        )
16520        .await;
16521        register_capability_manifest(
16522            &handler,
16523            &opener_ctx,
16524            &mut opener_rx,
16525            capability_manifest("opener", &[], &[]),
16526            2,
16527        )
16528        .await;
16529        let (mut client_rx, _) = open_route_for_capability_test(
16530            &handler,
16531            &target_ctx,
16532            &mut target_rx,
16533            732,
16534            3,
16535            "target",
16536            Some(ConsumerIdentity {
16537                module_id: "opener".to_string(),
16538                launch_nonce: "opener-nonce".to_string(),
16539            }),
16540        )
16541        .await;
16542        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16543
16544        handler
16545            .handle_control_frame(
16546                &target_ctx,
16547                catalog_update_with_capabilities_frame(
16548                    4,
16549                    CapabilityDeclarations {
16550                        provides: Vec::new(),
16551                        requires: Vec::new(),
16552                        must_never_reach: Vec::new(),
16553                    },
16554                ),
16555            )
16556            .await
16557            .expect("claim removal catalog.update succeeds");
16558        assert_eq!(
16559            forwarding.active_binding_count().unwrap(),
16560            1,
16561            "removing an attested target claim must leave the route census unchanged"
16562        );
16563        assert!(
16564            client_rx.try_recv().is_err(),
16565            "claim removal must not emit route.closed or a client route GOODBYE"
16566        );
16567        assert!(
16568            target_rx.try_recv().is_err(),
16569            "claim removal must not send the target a route GOODBYE"
16570        );
16571    }
16572
16573    /// A direct client may open a route to a denied capability provider; this
16574    /// policy applies only to attested supervised module origins, not to direct clients.
16575    #[tokio::test]
16576    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16577        let registry = Arc::new(Registry::default());
16578        let forwarding = Arc::new(ForwardingTable::default());
16579        let supervisor = SupervisorHandle::new();
16580        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16581        let handler =
16582            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16583                .with_supervisor(supervisor);
16584        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16585        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16586        register_capability_manifest(
16587            &handler,
16588            &target_ctx,
16589            &mut target_rx,
16590            capability_manifest("target", &["credentials-provider/v1"], &[]),
16591            1,
16592        )
16593        .await;
16594        register_capability_manifest(
16595            &handler,
16596            &opener_ctx,
16597            &mut opener_rx,
16598            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16599            2,
16600        )
16601        .await;
16602
16603        let (_client_rx, bind) = open_route_for_capability_test(
16604            &handler,
16605            &target_ctx,
16606            &mut target_rx,
16607            742,
16608            3,
16609            "target",
16610            None,
16611        )
16612        .await;
16613        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16614            panic!("direct scope-honesty route must bind");
16615        };
16616        assert_eq!(principal, Some(Principal::Direct));
16617        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16618    }
16619
16620    /// A module that denies a capability receives no self-route exemption when it
16621    /// also attestedly provides that capability.
16622    #[tokio::test]
16623    async fn must_never_reach_self_route_is_capability_forbidden() {
16624        let registry = Arc::new(Registry::default());
16625        let forwarding = Arc::new(ForwardingTable::default());
16626        let supervisor = SupervisorHandle::new();
16627        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16628        let handler =
16629            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16630                .with_supervisor(supervisor);
16631        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16632        register_capability_manifest(
16633            &handler,
16634            &self_ctx,
16635            &mut self_rx,
16636            capability_manifest(
16637                "self-provider",
16638                &["credentials-provider/v1"],
16639                &["credentials-provider/v1"],
16640            ),
16641            1,
16642        )
16643        .await;
16644
16645        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16646        let replies = handler
16647            .handle_control_frame(
16648                &client_ctx,
16649                route_open_frame_with_admission_facts(
16650                    2,
16651                    "self-provider",
16652                    unique_project_root("admission-facts"),
16653                    Some(ConsumerIdentity {
16654                        module_id: "self-provider".to_string(),
16655                        launch_nonce: "self-nonce".to_string(),
16656                    }),
16657                    None,
16658                ),
16659            )
16660            .await
16661            .expect("self-route refusal returns a typed frame");
16662        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16663        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16664        assert!(
16665            self_rx.try_recv().is_err(),
16666            "self denial must not relay route.bind"
16667        );
16668    }
16669
16670    #[test]
16671    fn unsupported_channel_zero_frame_returns_error() {
16672        let handler = ControlHandler::default();
16673        let request = Frame::build(
16674            FrameType::Request,
16675            control_flags(),
16676            0,
16677            0,
16678            21,
16679            b"opaque".to_vec(),
16680        )
16681        .unwrap();
16682
16683        let response = handler
16684            .handle_control(ConnectionId::new(1), request)
16685            .unwrap();
16686
16687        assert_eq!(response[0].header.ty, FrameType::Error);
16688        assert_eq!(
16689            parse_error(&response[0])["code"],
16690            "unsupported_control_frame"
16691        );
16692    }
16693
16694    /// Blue/green swap at the control-plane boundary. The supervisor that opens
16695    /// a swap is not wired yet, so the candidate is registered here directly
16696    /// into the registry and forwarding candidate slots, the way the swap's
16697    /// HELLO admission will.
16698    mod swap {
16699        use super::*;
16700
16701        const INCUMBENT: ConnectionId = ConnectionId::new(30);
16702        const CANDIDATE: ConnectionId = ConnectionId::new(40);
16703
16704        struct Swap {
16705            registry: Arc<Registry>,
16706            forwarding: Arc<ForwardingTable>,
16707            handler: ControlHandler,
16708            incumbent_ctx: RouteCtx,
16709            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16710            candidate_ctx: RouteCtx,
16711            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16712        }
16713
16714        async fn swap_with_incumbent() -> Swap {
16715            let registry = Arc::new(Registry::default());
16716            let forwarding = Arc::new(ForwardingTable::default());
16717            let handler =
16718                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
16719            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16720            hello_via_sink(
16721                &handler,
16722                &incumbent_ctx,
16723                &mut incumbent_rx,
16724                hello_frame("aft", PROTOCOL_VERSION, 7),
16725            )
16726            .await;
16727            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16728            Swap {
16729                registry,
16730                forwarding,
16731                handler,
16732                incumbent_ctx,
16733                incumbent_rx,
16734                candidate_ctx,
16735                candidate_rx,
16736            }
16737        }
16738
16739        fn register_candidate(swap: &Swap, ready: Option<bool>) {
16740            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16741            candidate_manifest.ready = ready;
16742            let registration = swap
16743                .registry
16744                .register_candidate_with_control_ops(
16745                    candidate_manifest,
16746                    PROTOCOL_VERSION,
16747                    CANDIDATE,
16748                    module_baseline_control_ops(),
16749                )
16750                .unwrap();
16751            swap.forwarding
16752                .register_candidate_module_connection(
16753                    CANDIDATE,
16754                    "aft".to_string(),
16755                    PROTOCOL_VERSION,
16756                    manifest_concurrency(&registration.manifest),
16757                    swap.candidate_ctx.egress.clone(),
16758                )
16759                .unwrap();
16760        }
16761
16762        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16763            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16764            swap.registry.promote_candidate("aft").unwrap().unwrap();
16765            cutover.incumbent.unwrap()
16766        }
16767
16768        /// A swap candidate's HELLO reserves its candidate registry entry
16769        /// before installing its candidate forwarding endpoint. The swap
16770        /// supervisor promotes a ready candidate as soon as it reads it as
16771        /// registered, and promotion cuts forwarding over to the candidate
16772        /// endpoint, so the lookup the supervisor reads
16773        /// (`Registry::published_registration`) must not return the candidate
16774        /// until that endpoint exists. Otherwise the cutover finds no candidate
16775        /// endpoint and the swap fails as if the candidate had disconnected.
16776        #[tokio::test]
16777        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16778            let mut swap = swap_with_incumbent().await;
16779            let hold = Arc::new(std::sync::Barrier::new(2));
16780            let handler = swap
16781                .handler
16782                .clone()
16783                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16784            handler
16785                .supervisor
16786                .open_swap("aft", "candidate-nonce".to_string());
16787            let hello = {
16788                let handler = handler.clone();
16789                let candidate_ctx = swap.candidate_ctx.clone();
16790                std::thread::spawn(move || {
16791                    handler
16792                        .handle_hello(
16793                            candidate_ctx.connection_id,
16794                            Some(candidate_ctx.egress.clone()),
16795                            hello_frame_with_nonce(
16796                                "aft",
16797                                PROTOCOL_VERSION,
16798                                8,
16799                                Some("candidate-nonce"),
16800                            ),
16801                        )
16802                        .unwrap()
16803                })
16804            };
16805
16806            hold.wait();
16807            // Parked: the candidate slot is claimed but not published, and
16808            // forwarding has no candidate endpoint.
16809            assert!(swap
16810                .registry
16811                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16812                .unwrap()
16813                .is_some());
16814            assert!(swap
16815                .registry
16816                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16817                .unwrap()
16818                .is_none());
16819            assert!(swap
16820                .forwarding
16821                .module_endpoint_for_connection(CANDIDATE)
16822                .unwrap()
16823                .is_none());
16824
16825            hold.wait();
16826            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16827            assert_eq!(
16828                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16829                FrameType::HelloAck
16830            );
16831            let published = swap
16832                .registry
16833                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16834                .unwrap()
16835                .expect("candidate is published once its HELLO completes");
16836            assert_eq!(published.connection_id, CANDIDATE);
16837            assert!(swap
16838                .forwarding
16839                .module_endpoint_for_connection(CANDIDATE)
16840                .unwrap()
16841                .is_some());
16842        }
16843
16844        fn keyed_total(counters: &Value, key: &str) -> u64 {
16845            counters[key]
16846                .as_object()
16847                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16848                .unwrap_or(0)
16849        }
16850
16851        #[tokio::test]
16852        async fn replacement_logs_old_end_and_new_admission_once() {
16853            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16854            let mut swap = swap_with_incumbent().await;
16855            swap.handler
16856                .supervisor
16857                .open_swap("aft", "candidate-nonce".to_string());
16858            hello_via_sink(
16859                &swap.handler,
16860                &swap.candidate_ctx,
16861                &mut swap.candidate_rx,
16862                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16863            )
16864            .await;
16865            cutover(&swap);
16866            swap.handler.cleanup_connection(INCUMBENT).unwrap();
16867
16868            let captured = crate::router::test_log::captured_logs(&logs);
16869            println!("captured replacement registry logs:\n{captured}");
16870            let new_admissions: Vec<_> = captured
16871                .lines()
16872                .filter(|line| {
16873                    line.contains("connection_id=40")
16874                        && line.contains("swap candidate registered; not routable until cutover")
16875                })
16876                .collect();
16877            assert_eq!(
16878                new_admissions.len(),
16879                1,
16880                "unexpected admission logs: {captured}"
16881            );
16882            assert!(new_admissions[0].contains("module_id=aft"));
16883            assert!(new_admissions[0].contains("connection_id=40"));
16884            assert!(
16885                new_admissions[0].contains("swap candidate registered; not routable until cutover")
16886            );
16887            assert!(new_admissions[0].contains("ready=true"));
16888
16889            let old_admissions: Vec<_> = captured
16890                .lines()
16891                .filter(|line| {
16892                    line.contains("module registered module_id=aft ")
16893                        && line.contains("connection_id=30")
16894                })
16895                .collect();
16896            assert_eq!(
16897                old_admissions.len(),
16898                1,
16899                "unexpected incumbent admission: {captured}"
16900            );
16901
16902            let old_ends: Vec<_> = captured
16903                .lines()
16904                .filter(|line| {
16905                    line.contains("connection_id=30") && line.contains("module registration ended")
16906                })
16907                .collect();
16908            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16909            assert!(old_ends[0].contains("module_id=aft"));
16910            assert!(old_ends[0].contains("connection_id=30"));
16911            assert!(old_ends[0].contains("reason=replaced"));
16912            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16913            assert_eq!(
16914                captured
16915                    .lines()
16916                    .filter(|line| line.contains("module registration promoted"))
16917                    .count(),
16918                1,
16919                "unexpected promotion logs: {captured}"
16920            );
16921        }
16922
16923        /// An ack from the incumbent for a bind it was sent before cutover,
16924        /// arriving before the incumbent is drained. The incumbent is the live
16925        /// connection carrying every other client's routes, so the ack must
16926        /// not end it: the waiting client is told to retry, the reservation is
16927        /// given back, and the incumbent is told to drop just that binding.
16928        #[tokio::test]
16929        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16930            let mut swap = swap_with_incumbent().await;
16931            let handler = swap.handler.clone();
16932
16933            // A co-tenant route, bound on the incumbent before the swap.
16934            let cotenant = ConnectionId::new(31);
16935            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16936            let (cotenant_task, cotenant_bind) = relay_route_open(
16937                &handler,
16938                cotenant,
16939                &cotenant_ctx.egress,
16940                &mut swap.incumbent_rx,
16941                100,
16942                "aft",
16943                "swap-cotenant",
16944            )
16945            .await;
16946            handler
16947                .handle_control_frame(
16948                    &swap.incumbent_ctx,
16949                    route_bind_ack(cotenant_bind.header.corr),
16950                )
16951                .await
16952                .unwrap();
16953            assert!(cotenant_task.await.unwrap().is_empty());
16954            let (cotenant_channel, cotenant_epoch) =
16955                published_route(&cotenant_rx.recv().await.unwrap());
16956
16957            // A second route.open, relayed to the incumbent and not yet acked.
16958            let caller = ConnectionId::new(32);
16959            let (caller_ctx, mut caller_rx) = route_ctx(caller);
16960            let (caller_task, caller_bind) = relay_route_open(
16961                &handler,
16962                caller,
16963                &caller_ctx.egress,
16964                &mut swap.incumbent_rx,
16965                101,
16966                "aft",
16967                "swap-caller",
16968            )
16969            .await;
16970            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16971
16972            register_candidate(&swap, None);
16973            cutover(&swap);
16974
16975            // The incumbent acks after promotion and before any drain.
16976            let ack = handler
16977                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16978                .await;
16979            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16980            if module_loop_error.is_some() {
16981                // What the connection loop does with an untranslated router
16982                // error: end the connection, releasing every route on it.
16983                handler.cleanup_connection(INCUMBENT).unwrap();
16984            }
16985
16986            // 1. The incumbent's other routes survive.
16987            assert!(
16988                cotenant_rx.try_recv().is_err(),
16989                "the co-tenant route on the incumbent was torn down by one late ack: \
16990                 {module_loop_error:?}"
16991            );
16992            assert!(matches!(
16993                swap.forwarding
16994                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16995                    .unwrap(),
16996                DataRoute::Client(DataRouteState::Bound(_))
16997            ));
16998            assert_eq!(module_loop_error, None);
16999            assert!(swap
17000                .registry
17001                .get_module_by_connection(INCUMBENT)
17002                .unwrap()
17003                .is_some());
17004
17005            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
17006            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17007                .await
17008                .expect("the incumbent is told to drop the abandoned binding")
17009                .unwrap()
17010                .frame;
17011            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17012            assert_eq!(goodbye.header.channel, abandoned_channel);
17013            assert_eq!(goodbye.header.epoch, abandoned_epoch);
17014            assert!(swap.incumbent_rx.try_recv().is_err());
17015
17016            // 3. The waiting client gets a retryable refusal and no route.
17017            let response = caller_task.await.unwrap();
17018            assert_eq!(response.len(), 1);
17019            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17020            assert!(caller_rx.try_recv().is_err());
17021
17022            // 4. The reservation pair is given back, and the pending bind
17023            //    settled exactly once: one accepted open (the co-tenant) and one
17024            //    refused open (the caller), nothing counted twice.
17025            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17026            let counters = handler.counters().snapshot();
17027            assert_eq!(
17028                keyed_total(&counters, "route_open_accepted_by_principal"),
17029                1
17030            );
17031            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17032            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17033        }
17034
17035        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
17036        /// id would resolve to the promoted candidate and every new route.open
17037        /// would be refused as reloading, leaving neither process routable.
17038        #[tokio::test]
17039        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17040            let mut swap = swap_with_incumbent().await;
17041            register_candidate(&swap, None);
17042            let incumbent = cutover(&swap);
17043            swap.forwarding
17044                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17045                .unwrap()
17046                .expect("the incumbent is still registered");
17047
17048            let client = ConnectionId::new(33);
17049            let (client_ctx, mut client_rx) = route_ctx(client);
17050            let route_handler = swap.handler.clone();
17051            let open_ctx = RouteCtx {
17052                connection_id: client,
17053                egress: client_ctx.egress.clone(),
17054            };
17055            let mut route_task = tokio::spawn(async move {
17056                route_handler
17057                    .handle_control_frame(
17058                        &open_ctx,
17059                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17060                    )
17061                    .await
17062                    .unwrap()
17063            });
17064            let bind = tokio::select! {
17065                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17066                response = &mut route_task => {
17067                    let response = response.unwrap();
17068                    panic!(
17069                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
17070                        parse_error(&response[0])["code"]
17071                    );
17072                }
17073            };
17074            swap.handler
17075                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17076                .await
17077                .unwrap();
17078            assert!(route_task.await.unwrap().is_empty());
17079            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17080            match swap
17081                .forwarding
17082                .lookup_data_route(client, channel, epoch)
17083                .unwrap()
17084            {
17085                DataRoute::Client(DataRouteState::Bound(route)) => {
17086                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17087                }
17088                other => panic!("expected a bound route on the candidate, got {other:?}"),
17089            }
17090            assert!(swap.incumbent_rx.try_recv().is_err());
17091        }
17092
17093        /// A candidate declares itself ready with `catalog.update` on its own
17094        /// connection. If the connection-keyed registry lookups searched only the
17095        /// active slot, this would answer `not_registered` and the candidate
17096        /// would never become ready.
17097        #[tokio::test]
17098        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17099            let swap = swap_with_incumbent().await;
17100            register_candidate(&swap, Some(false));
17101            let update = Frame::build(
17102                FrameType::Request,
17103                control_flags(),
17104                0,
17105                0,
17106                55,
17107                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17108                    provides: manifest("aft", PROTOCOL_VERSION).provides,
17109                    capabilities: None,
17110                    ready: Some(true),
17111                })
17112                .unwrap(),
17113            )
17114            .unwrap();
17115
17116            let replies = swap
17117                .handler
17118                .handle_control_frame(&swap.candidate_ctx, update)
17119                .await
17120                .unwrap();
17121
17122            assert_eq!(replies.len(), 1);
17123            assert_eq!(
17124                replies[0].header.ty,
17125                FrameType::Response,
17126                "candidate catalog.update was refused: {:?}",
17127                serde_json::from_slice::<Value>(&replies[0].body).ok()
17128            );
17129            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17130            assert_eq!(
17131                swap.registry
17132                    .get_module("aft")
17133                    .unwrap()
17134                    .unwrap()
17135                    .connection_id,
17136                INCUMBENT
17137            );
17138        }
17139    }
17140
17141    /// The HELLO gate while the supervisor has a swap open: only the nonce it
17142    /// minted for the candidate admits a second process, into the candidate
17143    /// slot, and that check runs ahead of the reserved-module gate.
17144    mod swap_admission {
17145        use super::*;
17146
17147        const INCUMBENT_NONCE: &str = "incumbent-nonce";
17148        const CANDIDATE_NONCE: &str = "candidate-nonce";
17149
17150        fn handler_with_incumbent(
17151            module_id: &str,
17152            reserved: bool,
17153        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17154            let registry = Arc::new(Registry::default());
17155            let supervisor = SupervisorHandle::new();
17156            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17157            if reserved {
17158                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17159            }
17160            let handler =
17161                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
17162            let incumbent = handler
17163                .handle_control(
17164                    ConnectionId::new(1),
17165                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17166                )
17167                .unwrap();
17168            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17169            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17170            (registry, supervisor, handler)
17171        }
17172
17173        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
17174        /// admits every nonce, so while a swap is open the swap gate is the only
17175        /// thing between a key-holder and the candidate slot. A nonce the
17176        /// supervisor did not mint, or none at all, is refused, and neither the
17177        /// incumbent's registration nor the candidate slot moves.
17178        #[test]
17179        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17180            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17181
17182            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17183                let replies = handler
17184                    .handle_control(
17185                        ConnectionId::new(connection),
17186                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17187                    )
17188                    .unwrap();
17189                assert_eq!(replies[0].header.ty, FrameType::Error);
17190                assert_eq!(
17191                    parse_error(&replies[0])["code"],
17192                    "swap_token_invalid",
17193                    "nonce {nonce:?}"
17194                );
17195            }
17196            assert!(registry.get_candidate("aft").unwrap().is_none());
17197            assert_eq!(
17198                registry.get_module("aft").unwrap().unwrap().connection_id,
17199                ConnectionId::new(1)
17200            );
17201
17202            // Control: the minted token is admitted, into the candidate slot,
17203            // and only once.
17204            let admitted = handler
17205                .handle_control(
17206                    ConnectionId::new(4),
17207                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17208                )
17209                .unwrap();
17210            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17211            assert_eq!(
17212                registry
17213                    .get_candidate("aft")
17214                    .unwrap()
17215                    .unwrap()
17216                    .connection_id,
17217                ConnectionId::new(4)
17218            );
17219            assert_eq!(
17220                registry.get_module("aft").unwrap().unwrap().connection_id,
17221                ConnectionId::new(1),
17222                "the candidate must not take the active slot"
17223            );
17224            let replayed = handler
17225                .handle_control(
17226                    ConnectionId::new(5),
17227                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17228                )
17229                .unwrap();
17230            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17231
17232            // The case only this gate covers: the incumbent has died mid-swap,
17233            // so its duplicate refusal is gone too, and without the gate a
17234            // key-holder would take the id's ACTIVE slot.
17235            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17236            let squatter = handler
17237                .handle_control(
17238                    ConnectionId::new(6),
17239                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17240                )
17241                .unwrap();
17242            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17243            assert!(
17244                registry.get_module("aft").unwrap().is_none(),
17245                "a squatter took the active slot of an id being swapped"
17246            );
17247        }
17248
17249        /// Design mutation arm (iii). A reserved module's candidate presents a
17250        /// nonce the reserved gate has never seen (that gate holds the
17251        /// incumbent's), so the swap gate must run first or the candidate is
17252        /// refused `reserved_module` and a reserved module can never be swapped.
17253        #[test]
17254        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17255            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17256
17257            let replies = handler
17258                .handle_control(
17259                    ConnectionId::new(2),
17260                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17261                )
17262                .unwrap();
17263
17264            assert_eq!(
17265                replies[0].header.ty,
17266                FrameType::HelloAck,
17267                "reserved candidate refused: {:?}",
17268                serde_json::from_slice::<Value>(&replies[0].body).ok()
17269            );
17270            assert_eq!(
17271                registry
17272                    .get_candidate("vault")
17273                    .unwrap()
17274                    .unwrap()
17275                    .connection_id,
17276                ConnectionId::new(2)
17277            );
17278        }
17279
17280        /// With no swap open the gate is inert: the incumbent's reserved gate
17281        /// and duplicate refusal behave exactly as before.
17282        #[test]
17283        fn without_an_open_swap_the_ordinary_gates_decide() {
17284            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17285            supervisor.close_swap("vault");
17286
17287            let candidate = handler
17288                .handle_control(
17289                    ConnectionId::new(2),
17290                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17291                )
17292                .unwrap();
17293            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17294            let duplicate = handler
17295                .handle_control(
17296                    ConnectionId::new(3),
17297                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17298                )
17299                .unwrap();
17300            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17301            assert!(registry.get_candidate("vault").unwrap().is_none());
17302        }
17303    }
17304
17305    /// `scope.sync` and `scope.describe` through the real control handler: who
17306    /// may sync is decided by the registration and launch nonce of the module
17307    /// connection, never by the request body.
17308    mod scopes {
17309        mod idle_tests {
17310            use super::*;
17311            include!("scope_idle_tests.rs");
17312        }
17313        mod route_open_timings {
17314            use super::*;
17315            include!("route_open_timing_tests.rs");
17316        }
17317        mod route_open_refusal_log {
17318            use super::*;
17319            include!("route_open_refusal_log_tests.rs");
17320        }
17321        mod activity {
17322            include!("activity_scope_tests.rs");
17323        }
17324        mod delta {
17325            include!("scope_handler_delta_tests.rs");
17326        }
17327        mod call_modes {
17328            include!("scope_call_mode_tests.rs");
17329        }
17330        #[tokio::test]
17331        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17332            let rig = rig().await;
17333            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17334                "s".into(),
17335                1,
17336                "tool".into(),
17337            ))
17338            .unwrap();
17339            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17340            let (direct, _rx) = wide_ctx(999);
17341            for (ctx, expected) in [
17342                (&rig.owner, FrameType::Response),
17343                (&direct, FrameType::Error),
17344            ] {
17345                let frame =
17346                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17347                        .unwrap();
17348                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17349                assert_eq!(replies[0].header.ty, expected);
17350                if expected == FrameType::Response {
17351                    let reply: subc_control::ScopeServedByReply =
17352                        serde_json::from_slice(&replies[0].body).unwrap();
17353                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17354                } else {
17355                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17356                }
17357            }
17358            let reply = call(
17359                &rig.handler,
17360                &rig.owner,
17361                &ModuleControlRequestFromModule::ScopeDescribe {
17362                    owner: Principal::Reserved {
17363                        module_id: OWNER.into(),
17364                    },
17365                    scope_ref: "s".into(),
17366                },
17367            )
17368            .await;
17369            assert!(matches!(
17370                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17371                ModuleControlResponseToModule::ScopeDescribe { .. }
17372            ));
17373        }
17374
17375        use subc_protocol::scope::{
17376            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17377            ScopeStatus,
17378        };
17379
17380        use super::*;
17381
17382        const OWNER: &str = "prefrontal-core";
17383
17384        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17385            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17386        }
17387
17388        async fn call(
17389            handler: &ControlHandler,
17390            ctx: &RouteCtx,
17391            request: &ModuleControlRequestFromModule,
17392        ) -> Frame {
17393            let body = serde_json::to_vec(request).unwrap();
17394            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17395            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17396            assert_eq!(replies.len(), 1, "{replies:?}");
17397            replies.pop().unwrap()
17398        }
17399
17400        async fn sync(
17401            handler: &ControlHandler,
17402            ctx: &RouteCtx,
17403            generation: u64,
17404            scopes: Vec<ScopeRecord>,
17405        ) -> Result<ModuleControlResponseToModule, String> {
17406            let reply = call(
17407                handler,
17408                ctx,
17409                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17410            )
17411            .await;
17412            match reply.header.ty {
17413                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17414                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17415            }
17416        }
17417
17418        async fn describe(
17419            handler: &ControlHandler,
17420            ctx: &RouteCtx,
17421            owner: &str,
17422            scope_ref: &str,
17423        ) -> ModuleControlResponseToModule {
17424            let reply = call(
17425                handler,
17426                ctx,
17427                &ModuleControlRequestFromModule::ScopeDescribe {
17428                    owner: Principal::Reserved {
17429                        module_id: owner.to_string(),
17430                    },
17431                    scope_ref: scope_ref.to_string(),
17432                },
17433            )
17434            .await;
17435            assert_eq!(
17436                reply.header.ty,
17437                FrameType::Response,
17438                "{:?}",
17439                parse_error(&reply)
17440            );
17441            serde_json::from_slice(&reply.body).unwrap()
17442        }
17443
17444        #[tokio::test]
17445        async fn scope_describe_never_echoes_bind_proof_or_hash() {
17446            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17447            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17448            let mut rig = rig().await;
17449            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17450                .await;
17451            let _bound = rig
17452                .bound(
17453                    Some(AFT),
17454                    PLEXUS,
17455                    Some(
17456                        rig.selector("s", Some(1))
17457                            .with_bind_proof(Some(PROOF.into())),
17458                    ),
17459                )
17460                .await;
17461            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17462            let output = serde_json::to_string(&reply).unwrap();
17463            assert!(output.contains("\"status\":\"live\""));
17464            assert!(!output.contains(PROOF));
17465            assert!(!output.contains(HASH));
17466        }
17467
17468        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
17469        async fn module(
17470            handler: &ControlHandler,
17471            connection: u64,
17472            module_id: &str,
17473            nonce: Option<&str>,
17474        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17475            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17476            hello_via_sink(
17477                handler,
17478                &ctx,
17479                &mut rx,
17480                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17481            )
17482            .await;
17483            (ctx, rx)
17484        }
17485
17486        /// `direct` and every other client connection has no registration, so
17487        /// it can neither sync nor own a scope.
17488        #[tokio::test]
17489        async fn a_client_connection_cannot_sync_or_describe() {
17490            let handler = ControlHandler::new(Arc::new(Registry::default()));
17491            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17492            for request in [
17493                ModuleControlRequestFromModule::ScopeSync {
17494                    generation: 1,
17495                    scopes: vec![head("s", 1)],
17496                },
17497                ModuleControlRequestFromModule::ScopeDescribe {
17498                    owner: Principal::Direct,
17499                    scope_ref: "s".to_string(),
17500                },
17501            ] {
17502                let reply = call(&handler, &ctx, &request).await;
17503                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17504            }
17505            assert!(
17506                !handler
17507                    .scopes
17508                    .read()
17509                    .unwrap()
17510                    .describe(
17511                        &Principal::Reserved {
17512                            module_id: OWNER.to_string()
17513                        },
17514                        "s"
17515                    )
17516                    .owner_synced
17517            );
17518        }
17519
17520        /// A module the supervisor did not spawn registers without a launch
17521        /// nonce, so it is never an owner's current launch.
17522        #[tokio::test]
17523        async fn a_module_without_a_supervised_launch_cannot_sync() {
17524            let handler = ControlHandler::new(Arc::new(Registry::default()));
17525            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17526            assert_eq!(
17527                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17528                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17529            );
17530        }
17531
17532        #[tokio::test]
17533        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17534            let supervisor = SupervisorHandle::new();
17535            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17536            let handler = ControlHandler::new(Arc::new(Registry::default()))
17537                .with_supervisor(supervisor.clone());
17538            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17539            sync(&handler, &incumbent, 1, vec![head("s", 1)])
17540                .await
17541                .expect("the current launch syncs");
17542
17543            // A swap candidate registers with the swap token and is refused
17544            // while the incumbent keeps syncing.
17545            supervisor.open_swap(OWNER, "n2".to_string());
17546            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17547            assert_eq!(
17548                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17549                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17550            );
17551            sync(&handler, &incumbent, 2, vec![head("s", 1)])
17552                .await
17553                .expect("the serving owner syncs during the swap");
17554
17555            // The swap fails and is rolled back. The candidate never held sync
17556            // authority, and still cannot sync.
17557            supervisor.close_swap(OWNER);
17558            assert_eq!(
17559                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17560                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17561            );
17562            sync(&handler, &incumbent, 3, vec![head("s", 1)])
17563                .await
17564                .expect("the serving owner syncs after the rollback");
17565            handler.cleanup_connection(candidate.connection_id).unwrap();
17566
17567            // A swap that cuts over. Promotion records the candidate's nonce as
17568            // the module's spawn nonce, which is what `set_spawn_nonce` does
17569            // here; the promoted connection then takes authority at any
17570            // generation and the superseded incumbent is refused.
17571            supervisor.open_swap(OWNER, "n3".to_string());
17572            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17573            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17574            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17575                .await
17576                .expect("the promoted launch takes authority");
17577            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17578                panic!("unexpected reply {reply:?}");
17579            };
17580            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17581            assert_eq!(
17582                sync(&handler, &incumbent, 4, Vec::new()).await,
17583                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17584            );
17585        }
17586
17587        /// Authority dies with its connection: the cleanup path releases it,
17588        /// so the owner's next connection takes it at any generation.
17589        #[tokio::test]
17590        async fn closing_the_authority_connection_frees_sync_authority() {
17591            let supervisor = SupervisorHandle::new();
17592            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17593            let handler = ControlHandler::new(Arc::new(Registry::default()))
17594                .with_supervisor(supervisor.clone());
17595            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17596            sync(&handler, &first, 10, vec![head("s", 1)])
17597                .await
17598                .unwrap();
17599            handler.cleanup_connection(first.connection_id).unwrap();
17600
17601            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17602            sync(&handler, &second, 1, vec![head("s", 1)])
17603                .await
17604                .expect("the next connection takes the released authority");
17605        }
17606
17607        #[tokio::test]
17608        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17609            let supervisor = SupervisorHandle::new();
17610            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17611            let handler =
17612                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17613            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17614            sync(&handler, &first, 10, vec![head("s", 1)])
17615                .await
17616                .unwrap();
17617            handler
17618                .handle_control_frame(
17619                    &first,
17620                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17621                )
17622                .await
17623                .unwrap();
17624            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17625            sync(&handler, &second, 1, vec![head("s", 1)])
17626                .await
17627                .expect("GOODBYE releases authority even if the old socket remains open");
17628        }
17629
17630        #[tokio::test]
17631        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17632            let registry = Arc::new(Registry::default());
17633            let supervisor_handle = SupervisorHandle::new();
17634            let supervisor =
17635                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17636                    .with_handle(supervisor_handle.clone())
17637                    .with_daemon_incarnation("incarnation-7".to_string());
17638            // Configured with enabled: false, so the supervisor lists the
17639            // module without spawning a process for it.
17640            supervisor
17641                .supervise_configured(
17642                    ModuleSpec {
17643                        carry_containment: false,
17644                        module_id: OWNER.to_string(),
17645                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17646                        args: Vec::new(),
17647                        env: Vec::new(),
17648                        reserved: false,
17649                        reserved_prefixes: Vec::new(),
17650                        protocol: ModuleProtocol::Subc,
17651                        overlap: Default::default(),
17652                    },
17653                    false,
17654                )
17655                .unwrap();
17656            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17657            let handler =
17658                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17659            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17660
17661            // Configured but not yet synced: a reader waits for the owner.
17662            let ModuleControlResponseToModule::ScopeDescribe {
17663                status,
17664                daemon_incarnation,
17665                owner_synced,
17666                owner_configured,
17667                scope,
17668                ..
17669            } = describe(&handler, &reader, OWNER, "s").await
17670            else {
17671                panic!("not a describe reply");
17672            };
17673            assert_eq!(status, ScopeStatus::NotLive);
17674            assert_eq!(daemon_incarnation, "incarnation-7");
17675            assert!(!owner_synced);
17676            assert!(owner_configured);
17677            assert!(scope.is_none());
17678
17679            // Not a supervised module: the owner will never sync, and a reader
17680            // refuses rather than waits.
17681            let ModuleControlResponseToModule::ScopeDescribe {
17682                status,
17683                owner_configured,
17684                ..
17685            } = describe(&handler, &reader, "ghost", "s").await
17686            else {
17687                panic!("not a describe reply");
17688            };
17689            assert_eq!(status, ScopeStatus::NotLive);
17690            assert!(!owner_configured);
17691
17692            // Live, with the stamp fields and the computed owner_authorized.
17693            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17694            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17695            let ModuleControlResponseToModule::ScopeDescribe {
17696                status,
17697                scope_epoch,
17698                owner_synced,
17699                scope,
17700                ..
17701            } = describe(&handler, &reader, OWNER, "s").await
17702            else {
17703                panic!("not a describe reply");
17704            };
17705            assert_eq!(status, ScopeStatus::Live);
17706            assert_eq!(scope_epoch, Some(4));
17707            assert!(owner_synced);
17708            let stamp = scope.expect("a live scope carries its stamp");
17709            assert!(
17710                stamp.owner_authorized,
17711                "prefrontal-core is the default authority"
17712            );
17713            assert_eq!(stamp.kind, ScopeKind::Head);
17714        }
17715
17716        #[tokio::test]
17717        async fn scope_authority_owners_decides_owner_authorized() {
17718            let supervisor = SupervisorHandle::new();
17719            supervisor.set_spawn_nonce("broca", "b1".to_string());
17720            let handler = ControlHandler::new(Arc::new(Registry::default()))
17721                .with_supervisor(supervisor)
17722                .with_scope_authority_owners(vec!["broca".to_string()]);
17723            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17724            let mut gated = head("s", 1);
17725            gated.attributes.agent_id = Some("agent".to_string());
17726            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17727            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17728                describe(&handler, &broca, "broca", "s").await
17729            else {
17730                panic!("not a describe reply");
17731            };
17732            assert!(scope.unwrap().owner_authorized);
17733        }
17734
17735        /// With route admission, the stamp, the commit re-check and drains in
17736        /// place, the feature is advertised: the module ops in HELLO_ACK, and
17737        /// `scopes/v1` in HELLO_ACK and `server.describe`.
17738        #[tokio::test]
17739        async fn scope_ops_and_the_scopes_capability_are_advertised() {
17740            let handler = ControlHandler::new(Arc::new(Registry::default()));
17741            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17742            let ack = hello_via_sink(
17743                &handler,
17744                &ctx,
17745                &mut rx,
17746                hello_frame("m", PROTOCOL_VERSION, 1),
17747            )
17748            .await;
17749            let ack = parse_ack(&ack);
17750            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17751                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17752            }
17753            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17754
17755            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17756            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17757            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17758            let reply = handler
17759                .handle_control_frame(&client, frame)
17760                .await
17761                .unwrap()
17762                .pop()
17763                .unwrap();
17764            let ClientControlResponse::ServerDescribe { capabilities, .. } =
17765                serde_json::from_slice(&reply.body).unwrap()
17766            else {
17767                panic!("not a server.describe reply");
17768            };
17769            assert!(
17770                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17771                "{capabilities:?}"
17772            );
17773        }
17774
17775        // ---- route admission, stamps, commit re-check and drains ----------
17776
17777        const PLEXUS: &str = "plexus";
17778        const OTHER: &str = "other";
17779        const AFT: &str = "aft";
17780        const BROCA: &str = "broca";
17781        const MAGIC: &str = "magic-context";
17782
17783        fn nonce(module_id: &str) -> String {
17784            format!("nonce-{module_id}")
17785        }
17786
17787        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17788            let (tx, rx) = mpsc::channel(64);
17789            (
17790                RouteCtx {
17791                    connection_id: ConnectionId::new(connection),
17792                    egress: FrameSink::new(tx),
17793                },
17794                rx,
17795            )
17796        }
17797
17798        /// A daemon with a configured owner (prefrontal-core) registered on its
17799        /// own module connection, two routable targets (plexus, other), and
17800        /// launch nonces minted for the modules that open routes as carriers.
17801        struct Rig {
17802            handler: ControlHandler,
17803            forwarding: Arc<ForwardingTable>,
17804            owner: RouteCtx,
17805            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17806            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17807            generation: u64,
17808            next_connection: u64,
17809            _supervisor: Supervisor,
17810        }
17811
17812        async fn rig() -> Rig {
17813            rig_with_flow_support(true).await
17814        }
17815
17816        async fn rig_with_flow_support(flow_support: bool) -> Rig {
17817            rig_with_scope_capabilities(flow_support, false).await
17818        }
17819
17820        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17821            let mut provides = Vec::new();
17822            if flow_support {
17823                provides.push("flow-scopes/v1");
17824            }
17825            if run_support {
17826                provides.push("agent-run-scopes/v1");
17827            }
17828            rig_with_target_capabilities(&provides).await
17829        }
17830
17831        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17832            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17833            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17834            // Decoder versions alone cannot admit scoped calls: only a capability
17835            // declaration promises the required run or flow behaviour.
17836            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17837            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17838            Frame::build(
17839                FrameType::Hello,
17840                control_flags(),
17841                0,
17842                0,
17843                connection,
17844                serde_json::to_vec(&body).unwrap(),
17845            )
17846            .unwrap()
17847        }
17848
17849        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17850            let registry = Arc::new(Registry::default());
17851            let forwarding = Arc::new(ForwardingTable::default());
17852            let supervisor_handle = SupervisorHandle::new();
17853            let supervisor =
17854                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17855                    .with_handle(supervisor_handle.clone());
17856            supervisor
17857                .supervise_configured(
17858                    ModuleSpec {
17859                        carry_containment: false,
17860                        module_id: OWNER.to_string(),
17861                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17862                        args: Vec::new(),
17863                        env: Vec::new(),
17864                        reserved: false,
17865                        reserved_prefixes: Vec::new(),
17866                        protocol: ModuleProtocol::Subc,
17867                        overlap: Default::default(),
17868                    },
17869                    false,
17870                )
17871                .unwrap();
17872            for module_id in [OWNER, AFT, BROCA, MAGIC] {
17873                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17874            }
17875            let handler =
17876                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17877                    .with_supervisor(supervisor_handle);
17878            let (owner, mut owner_rx) = wide_ctx(1);
17879            hello_via_sink(
17880                &handler,
17881                &owner,
17882                &mut owner_rx,
17883                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17884            )
17885            .await;
17886            let mut modules = BTreeMap::new();
17887            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17888                let (ctx, mut rx) = wide_ctx(connection);
17889                let hello = hello_with_capabilities(module_id, connection, provides);
17890                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17891                modules.insert(module_id.to_string(), (ctx, rx));
17892            }
17893            Rig {
17894                handler,
17895                forwarding,
17896                owner,
17897                _owner_rx: owner_rx,
17898                modules,
17899                generation: 0,
17900                next_connection: 100,
17901                _supervisor: supervisor,
17902            }
17903        }
17904
17905        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17906            ScopeCarrier::new(Principal::Reserved {
17907                module_id: module_id.to_string(),
17908            })
17909            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17910        }
17911
17912        /// The scope most tests open under: aft carries to any module, broca
17913        /// only to plexus and other, and the owner delegates as agent-1.
17914        fn session(scope_epoch: u64) -> ScopeRecord {
17915            let mut record = head("s", scope_epoch);
17916            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17917            record.attributes.agent_id = Some("agent-1".to_string());
17918            record.attributes.delegates = true;
17919            record
17920        }
17921
17922        impl Rig {
17923            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17924                self.generation += 1;
17925                sync(&self.handler, &self.owner, self.generation, scopes)
17926                    .await
17927                    .expect("the owner's sync is accepted");
17928            }
17929
17930            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17931                ScopeSelector::new(
17932                    Principal::Reserved {
17933                        module_id: OWNER.to_string(),
17934                    },
17935                    scope_ref.to_string(),
17936                    scope_epoch,
17937                )
17938            }
17939
17940            fn open_frame(
17941                &mut self,
17942                opener: Option<&str>,
17943                target: &str,
17944                scope: Option<ScopeSelector>,
17945            ) -> (
17946                RouteCtx,
17947                mpsc::Receiver<crate::router::OutboundFrame>,
17948                Frame,
17949            ) {
17950                self.next_connection += 1;
17951                let (ctx, rx) = wide_ctx(self.next_connection);
17952                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17953                    busy_refusals: None,
17954                    target: RouteTarget::ToolProvider {
17955                        module_id: target.to_string(),
17956                    },
17957                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
17958                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
17959                        module_id: module_id.to_string(),
17960                        launch_nonce: nonce(module_id),
17961                    }),
17962                    consumer_capabilities: None,
17963                    role_versions: None,
17964                    admission_facts: None,
17965                    scope,
17966                })
17967                .unwrap();
17968                let frame = Frame::build(
17969                    FrameType::Request,
17970                    control_flags(),
17971                    0,
17972                    0,
17973                    self.next_connection,
17974                    body,
17975                )
17976                .unwrap();
17977                (ctx, rx, frame)
17978            }
17979
17980            /// Open and expect a refusal before anything is relayed.
17981            async fn refused(
17982                &mut self,
17983                opener: Option<&str>,
17984                target: &str,
17985                scope: Option<ScopeSelector>,
17986            ) -> String {
17987                self.refusal_body(opener, target, scope).await["code"]
17988                    .as_str()
17989                    .unwrap()
17990                    .to_string()
17991            }
17992
17993            async fn refusal_body(
17994                &mut self,
17995                opener: Option<&str>,
17996                target: &str,
17997                scope: Option<ScopeSelector>,
17998            ) -> Value {
17999                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18000                let replies = tokio::time::timeout(
18001                    Duration::from_secs(2),
18002                    self.handler.handle_control_frame(&ctx, frame),
18003                )
18004                .await
18005                .expect("the open must be refused before waiting for a bind ack")
18006                .unwrap();
18007                assert_eq!(replies.len(), 1, "{replies:?}");
18008                assert_eq!(replies[0].header.ty, FrameType::Error);
18009                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18010                assert!(
18011                    module_rx.try_recv().is_err(),
18012                    "a refused open relays nothing"
18013                );
18014                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18015                parse_error(&replies[0])
18016            }
18017
18018            /// Start an open and return its task and the bind the target got.
18019            async fn relayed(
18020                &mut self,
18021                opener: Option<&str>,
18022                target: &str,
18023                scope: Option<ScopeSelector>,
18024            ) -> Relayed {
18025                self.relayed_with_legacy_root(opener, target, scope, None)
18026                    .await
18027            }
18028
18029            async fn relayed_with_legacy_root(
18030                &mut self,
18031                opener: Option<&str>,
18032                target: &str,
18033                scope: Option<ScopeSelector>,
18034                legacy_root: Option<&std::path::Path>,
18035            ) -> Relayed {
18036                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18037                if let Some(root) = legacy_root {
18038                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18039                    body["identity"]["project_root"] = serde_json::json!(root);
18040                    frame.body = serde_json::to_vec(&body).unwrap();
18041                }
18042                let handler = self.handler.clone();
18043                let task_ctx = ctx.clone();
18044                let task = tokio::spawn(async move {
18045                    handler
18046                        .handle_control_frame(&task_ctx, frame)
18047                        .await
18048                        .unwrap()
18049                });
18050                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18051                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18052                    .await
18053                    .expect("the target receives the relayed route.bind")
18054                    .unwrap()
18055                    .frame;
18056                Relayed {
18057                    target: target.to_string(),
18058                    client: ctx,
18059                    client_rx: rx,
18060                    task,
18061                    bind,
18062                }
18063            }
18064
18065            async fn ack(&self, relayed: &Relayed) {
18066                let (module, _) = &self.modules[&relayed.target];
18067                self.handler
18068                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18069                    .await
18070                    .unwrap();
18071            }
18072
18073            /// Open, ack and return the bound route.
18074            async fn bound(
18075                &mut self,
18076                opener: Option<&str>,
18077                target: &str,
18078                scope: Option<ScopeSelector>,
18079            ) -> Bound {
18080                let relayed = self.relayed(opener, target, scope).await;
18081                self.commit_relayed(relayed).await
18082            }
18083
18084            async fn bound_with_legacy_root(
18085                &mut self,
18086                opener: Option<&str>,
18087                target: &str,
18088                scope: Option<ScopeSelector>,
18089                root: &std::path::Path,
18090            ) -> Bound {
18091                let relayed = self
18092                    .relayed_with_legacy_root(opener, target, scope, Some(root))
18093                    .await;
18094                self.commit_relayed(relayed).await
18095            }
18096
18097            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18098                self.ack(&relayed).await;
18099                let Relayed {
18100                    target,
18101                    client,
18102                    mut client_rx,
18103                    task,
18104                    bind,
18105                } = relayed;
18106                assert!(
18107                    task.await.unwrap().is_empty(),
18108                    "the open is answered by commit"
18109                );
18110                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18111                Bound {
18112                    target,
18113                    client,
18114                    client_rx,
18115                    channel,
18116                    epoch,
18117                    bind,
18118                }
18119            }
18120
18121            fn live(&self, route: &Bound) -> bool {
18122                matches!(
18123                    self.forwarding
18124                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18125                        .unwrap(),
18126                    DataRoute::Client(DataRouteState::Bound(_))
18127                )
18128            }
18129        }
18130
18131        struct Relayed {
18132            target: String,
18133            client: RouteCtx,
18134            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18135            task: tokio::task::JoinHandle<Vec<Frame>>,
18136            bind: Frame,
18137        }
18138
18139        struct Bound {
18140            target: String,
18141            client: RouteCtx,
18142            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18143            channel: u16,
18144            epoch: u32,
18145            bind: Frame,
18146        }
18147
18148        impl Bound {
18149            /// The reason of the `route.closed` this client was sent, after
18150            /// checking it also got a GOODBYE on exactly this route.
18151            fn closed_reason(&mut self) -> RouteCloseReason {
18152                let mut reason = None;
18153                let mut goodbye = false;
18154                while let Ok(outbound) = self.client_rx.try_recv() {
18155                    let frame = outbound.frame;
18156                    match frame.header.ty {
18157                        FrameType::Goodbye => {
18158                            assert_eq!(
18159                                (frame.header.channel, frame.header.epoch),
18160                                (self.channel, self.epoch)
18161                            );
18162                            goodbye = true;
18163                        }
18164                        FrameType::Push => {
18165                            let ClientControlPush::RouteClosed {
18166                                reason: r,
18167                                module_id,
18168                                ..
18169                            } = serde_json::from_slice(&frame.body).unwrap()
18170                            else {
18171                                panic!("unexpected push");
18172                            };
18173                            assert_eq!(module_id, self.target);
18174                            reason = Some(r);
18175                        }
18176                        other => panic!("unexpected frame {other:?}"),
18177                    }
18178                }
18179                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18180                reason.expect("the client is told why the route closed")
18181            }
18182
18183            fn untouched(&mut self) -> bool {
18184                self.client_rx.try_recv().is_err()
18185            }
18186
18187            fn stamp(&self) -> Option<ScopeStamp> {
18188                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18189                    ModuleControlRequest::RouteBind { scope, .. } => scope,
18190                    other => panic!("expected a route.bind, got {other:?}"),
18191                }
18192            }
18193        }
18194
18195        #[tokio::test]
18196        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18197        ) {
18198            let mut rig = rig().await;
18199            let mut record = session(1);
18200            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18201            record.child_owners = vec![Principal::Reserved {
18202                module_id: MAGIC.to_string(),
18203            }];
18204            rig.sync(vec![record]).await;
18205            let scope = || Some(rig_selector("s", Some(1)));
18206
18207            // Admitted: the owner, a bare carrier to any module, a targeted
18208            // carrier to its listed module.
18209            rig.bound(Some(OWNER), PLEXUS, scope()).await;
18210            rig.bound(Some(AFT), OTHER, scope()).await;
18211            rig.bound(Some(BROCA), PLEXUS, scope()).await;
18212
18213            // Refused scope_not_carrier: a targeted carrier to an unlisted
18214            // module, a module that is not listed at all (a child owner is not
18215            // a carrier), and a direct key-holder.
18216            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18217                assert_eq!(
18218                    rig.refused(opener, target, scope()).await,
18219                    error_codes::SCOPE_NOT_CARRIER,
18220                    "{opener:?} -> {target}"
18221                );
18222            }
18223        }
18224
18225        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18226            ScopeSelector::new(
18227                Principal::Reserved {
18228                    module_id: OWNER.to_string(),
18229                },
18230                scope_ref.to_string(),
18231                scope_epoch,
18232            )
18233        }
18234
18235        #[tokio::test]
18236        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18237            let mut rig = rig().await;
18238            rig.sync(vec![session(1)]).await;
18239            for opener in [OWNER, AFT] {
18240                assert_eq!(
18241                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18242                        .await,
18243                    error_codes::SCOPE_EPOCH_REQUIRED,
18244                    "{opener}"
18245                );
18246            }
18247        }
18248
18249        #[tokio::test]
18250        async fn admission_separates_not_synced_not_live_and_ended() {
18251            let mut rig = rig().await;
18252            // Before the configured owner's first sync: retryable.
18253            let code = rig
18254                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18255                .await;
18256            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18257            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18258
18259            // An owner that is not configured will never sync: terminal.
18260            let ghost = ScopeSelector::new(
18261                Principal::Reserved {
18262                    module_id: "ghost".to_string(),
18263                },
18264                "s".to_string(),
18265                Some(1),
18266            );
18267            assert_eq!(
18268                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18269                error_codes::SCOPE_NOT_LIVE
18270            );
18271
18272            rig.sync(vec![session(2)]).await;
18273            assert_eq!(
18274                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18275                    .await,
18276                error_codes::SCOPE_NOT_LIVE
18277            );
18278            for epoch in [1, 3] {
18279                assert_eq!(
18280                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18281                        .await,
18282                    error_codes::SCOPE_ENDED,
18283                    "epoch {epoch}"
18284                );
18285            }
18286            // Control: the live epoch is admitted.
18287            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18288                .await;
18289        }
18290
18291        #[tokio::test]
18292        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18293            let mut rig = rig().await;
18294            rig.sync(vec![session(1)]).await;
18295            let route = rig
18296                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18297                .await;
18298            let stamp = route.stamp().expect("a scoped bind carries the stamp");
18299            assert_eq!(stamp.scope_ref, "s");
18300            assert_eq!(stamp.scope_epoch, 1);
18301            assert_eq!(stamp.kind, ScopeKind::Head);
18302            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18303            assert!(stamp.attributes.delegates);
18304            assert!(stamp.owner_authorized);
18305            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18306            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18307
18308            // broca owns a scope of its own on its own module connection; it is
18309            // not in scope_authority_owners, so its stamp is not authorized.
18310            let (broca, mut broca_rx) = wide_ctx(50);
18311            hello_via_sink(
18312                &rig.handler,
18313                &broca,
18314                &mut broca_rx,
18315                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18316            )
18317            .await;
18318            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18319                .await
18320                .unwrap();
18321            let own = ScopeSelector::new(
18322                Principal::Reserved {
18323                    module_id: BROCA.to_string(),
18324                },
18325                "b".to_string(),
18326                Some(1),
18327            );
18328            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18329            assert!(!route.stamp().unwrap().owner_authorized);
18330        }
18331
18332        #[tokio::test]
18333        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18334            let mut rig = rig().await;
18335            let mut record = head("s", 1);
18336            record.carriers = vec![carrier(AFT, None)];
18337            let flow_id = "Flow:run-7/step_2!~";
18338            record.attributes.flow_id = Some(flow_id.to_string());
18339            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18340                .await
18341                .unwrap();
18342            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18343                panic!("not a sync reply");
18344            };
18345            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18346            let route = rig
18347                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18348                .await;
18349            assert!(rig.live(&route), "the stamped bind committed");
18350            let stamp = route.stamp().expect("a flow scope carries a stamp");
18351            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18352            assert_eq!(stamp.attributes.agent_id, None);
18353            assert!(!stamp.attributes.delegates);
18354            assert!(stamp.owner_authorized);
18355            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18356            assert_eq!(unscoped.stamp(), None);
18357        }
18358
18359        #[tokio::test]
18360        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18361            let mut rig = rig_with_flow_support(false).await;
18362            let mut record = session(1);
18363            record.attributes.flow_id = Some("flow:7".to_string());
18364            rig.sync(vec![record]).await;
18365            for opener in [OWNER, AFT] {
18366                let body = rig
18367                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18368                    .await;
18369                assert_eq!(body["code"], "target_flow_unsupported");
18370                let message = body["message"].as_str().unwrap();
18371                for required in [PLEXUS, "flow-scopes/v1"] {
18372                    assert!(message.contains(required), "{message}");
18373                }
18374            }
18375        }
18376
18377        #[tokio::test]
18378        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18379            let mut rig = rig_with_flow_support(true).await;
18380            let mut record = session(1);
18381            record.attributes.flow_id = Some("flow:7".to_string());
18382            rig.sync(vec![record]).await;
18383            for opener in [OWNER, AFT] {
18384                let route = rig
18385                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18386                    .await;
18387                assert!(rig.live(&route));
18388                assert_eq!(
18389                    route.stamp().unwrap().attributes.flow_id.as_deref(),
18390                    Some("flow:7")
18391                );
18392            }
18393        }
18394
18395        #[tokio::test]
18396        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18397            use std::future::Future;
18398
18399            let mut rig = rig_with_flow_support(true).await;
18400            let mut record = session(1);
18401            record.attributes.flow_id = Some("flow:7".to_string());
18402            rig.sync(vec![record]).await;
18403            let (client, mut client_rx, frame) =
18404                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18405            // Hold the route-open response permit so admission sees the first
18406            // target but relay reservation cannot capture an endpoint yet.
18407            for _ in 0..64 {
18408                client.egress.try_send(route_bind_ack(1)).unwrap();
18409            }
18410            let handler = rig.handler.clone();
18411            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18412            std::future::poll_fn(|cx| {
18413                assert!(open.as_mut().poll(cx).is_pending());
18414                std::task::Poll::Ready(())
18415            })
18416            .await;
18417            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18418
18419            let old_connection = rig.modules[PLEXUS].0.connection_id;
18420            rig.handler.cleanup_connection(old_connection).unwrap();
18421            let (replacement, mut replacement_rx) = wide_ctx(200);
18422            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18423            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18424            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18425            let hello = Frame::build(
18426                FrameType::Hello,
18427                control_flags(),
18428                0,
18429                0,
18430                200,
18431                serde_json::to_vec(&body).unwrap(),
18432            )
18433            .unwrap();
18434            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18435
18436            client_rx.try_recv().unwrap();
18437            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18438                .await
18439                .expect("the replacement is refused without waiting for a bind ack")
18440                .unwrap();
18441            assert_eq!(replies.len(), 1);
18442            let body = parse_error(&replies[0]);
18443            assert_eq!(body["code"], "target_flow_unsupported");
18444            for required in [PLEXUS, "flow-scopes/v1"] {
18445                assert!(body["message"].as_str().unwrap().contains(required));
18446            }
18447            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18448            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18449            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18450            assert!(rig
18451                .handler
18452                .registry
18453                .get_module_by_connection(replacement.connection_id)
18454                .unwrap()
18455                .is_some());
18456        }
18457
18458        #[tokio::test]
18459        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18460            use std::future::Future;
18461            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18462            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18463            const NEXT_HASH: &str =
18464                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18465            let mut rig = rig().await;
18466            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18467            record.carriers.push(ScopeCarrier::new(Principal::Direct));
18468            rig.sync(vec![record.clone()]).await;
18469            let (client, mut rx, frame) = rig.open_frame(
18470                None,
18471                PLEXUS,
18472                Some(
18473                    rig.selector("s", Some(1))
18474                        .with_bind_proof(Some(PROOF.into())),
18475                ),
18476            );
18477            // Hold egress reservation while the scope owner changes its bind-proof
18478            // hash, so a proof valid at admission must be checked again at relay.
18479            for _ in 0..64 {
18480                client.egress.try_send(route_bind_ack(1)).unwrap();
18481            }
18482            let handler = rig.handler.clone();
18483            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18484            std::future::poll_fn(|cx| {
18485                assert!(open.as_mut().poll(cx).is_pending());
18486                std::task::Poll::Ready(())
18487            })
18488            .await;
18489            record.bind_proof_sha256 = Some(NEXT_HASH.into());
18490            rig.sync(vec![record]).await;
18491            rx.try_recv().unwrap();
18492            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18493                .await
18494                .unwrap()
18495                .unwrap();
18496            assert_eq!(
18497                parse_error(&replies[0])["code"],
18498                error_codes::SCOPE_BIND_PROOF_MISMATCH
18499            );
18500            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18501            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18502            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18503        }
18504
18505        #[tokio::test]
18506        async fn scope_bind_proof_reserved_openers_ignore_proof() {
18507            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18508            let mut rig = rig().await;
18509            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18510                .await;
18511            for opener in [OWNER, AFT] {
18512                for proof in [None, Some("malformed+proof=")] {
18513                    let route = rig
18514                        .bound(
18515                            Some(opener),
18516                            PLEXUS,
18517                            Some(
18518                                rig.selector("s", Some(1))
18519                                    .with_bind_proof(proof.map(str::to_string)),
18520                            ),
18521                        )
18522                        .await;
18523                    assert!(rig.live(&route));
18524                }
18525            }
18526        }
18527
18528        #[tokio::test]
18529        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18530            use std::future::Future;
18531            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18532            const NEXT_HASH: &str =
18533                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18534            let mut rig = rig().await;
18535            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18536                .await;
18537            let (client, mut rx, frame) =
18538                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18539            for _ in 0..64 {
18540                client.egress.try_send(route_bind_ack(1)).unwrap();
18541            }
18542            let handler = rig.handler.clone();
18543            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18544            std::future::poll_fn(|cx| {
18545                assert!(open.as_mut().poll(cx).is_pending());
18546                std::task::Poll::Ready(())
18547            })
18548            .await;
18549            rig.sync(vec![
18550                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18551            ])
18552            .await;
18553            rx.try_recv().unwrap();
18554            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18555            let bind = std::future::poll_fn(|cx| {
18556                assert!(open.as_mut().poll(cx).is_pending());
18557                module_rx.poll_recv(cx)
18558            })
18559            .await
18560            .unwrap()
18561            .frame;
18562            handler
18563                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18564                .await
18565                .unwrap();
18566            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18567                .await
18568                .unwrap()
18569                .unwrap();
18570            assert_eq!(replies.len(), 1);
18571            // Proof rechecking allowed the relay. Independent tag settlement
18572            // still asks the opener to retry after any same-epoch content change.
18573            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18574            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18575            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18576            let route = rig
18577                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18578                .await;
18579            assert!(rig.live(&route));
18580        }
18581
18582        #[tokio::test]
18583        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18584        {
18585            let mut rig = rig_with_flow_support(false).await;
18586            rig.sync(vec![session(1)]).await;
18587            let route = rig
18588                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18589                .await;
18590            assert!(rig.live(&route));
18591            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18592            let mut record = session(1);
18593            record.attributes.flow_id = Some("flow:7".to_string());
18594            rig.sync(vec![record]).await;
18595            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18596            assert!(rig.live(&unscoped));
18597            assert_eq!(unscoped.stamp(), None);
18598        }
18599
18600        #[tokio::test]
18601        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18602            let mut rig = rig().await;
18603            let mut record = session(1);
18604            record.attributes.flow_id = Some("flow:7".to_string());
18605            rig.sync(vec![record.clone()]).await;
18606            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18607            let mut owner_route = rig
18608                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18609                .await;
18610            let mut carrier_route = rig
18611                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18612                .await;
18613            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18614            rig.sync(vec![record.clone()]).await;
18615            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18616            assert!(rig.live(&owner_route) && owner_route.untouched());
18617            assert!(rig.live(&carrier_route) && carrier_route.untouched());
18618
18619            record.attributes.flow_id = Some("flow:8".to_string());
18620            rig.sync(vec![record]).await;
18621            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18622            let before = before.unwrap();
18623            assert_eq!(after.scope_epoch, before.scope_epoch);
18624            assert!(after.version > before.version);
18625            for route in [&mut owner_route, &mut carrier_route] {
18626                assert!(!rig.live(route));
18627                assert_eq!(
18628                    route.closed_reason(),
18629                    RouteCloseReason::ScopeDelegationChanged
18630                );
18631            }
18632            assert!(rig.live(&unscoped) && unscoped.untouched());
18633            // Each provider also receives a GOODBYE for its drained route;
18634            // consume it before expecting the next route.bind on that sink.
18635            for target in [PLEXUS, OTHER] {
18636                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18637                let goodbye = module_rx
18638                    .try_recv()
18639                    .expect("the provider sees the drain")
18640                    .frame;
18641                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18642                assert!(module_rx.try_recv().is_err());
18643            }
18644            let rebound = rig
18645                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18646                .await;
18647            assert_eq!(
18648                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18649                Some("flow:8")
18650            );
18651        }
18652
18653        /// The owner's sync lands between admission and the module's ack. The
18654        /// open is refused by name, the module's other routes stay up, and the
18655        /// reserved pair is released. Changed content is retryable; an ended
18656        /// scope is not.
18657        #[tokio::test]
18658        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18659            let mut rig = rig().await;
18660            rig.sync(vec![session(1)]).await;
18661            let mut cotenant = rig
18662                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18663                .await;
18664
18665            let mut changed = session(1);
18666            changed.child_owners.push(Principal::Reserved {
18667                module_id: MAGIC.to_string(),
18668            });
18669            let mut ended = None;
18670            for (code, next) in [
18671                (error_codes::SCOPE_CHANGED, vec![changed]),
18672                (error_codes::SCOPE_ENDED, Vec::new()),
18673            ] {
18674                let relayed = rig
18675                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18676                    .await;
18677                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18678                ended = Some(next.is_empty());
18679                rig.sync(next).await;
18680                rig.ack(&relayed).await;
18681                let replies = relayed.task.await.unwrap();
18682                assert_eq!(replies.len(), 1, "{replies:?}");
18683                assert_eq!(parse_error(&replies[0])["code"], code);
18684                assert_eq!(
18685                    subc_protocol::error_codes::is_retryable_route_open(code),
18686                    code == error_codes::SCOPE_CHANGED
18687                );
18688                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18689                // The module is told to drop just the binding it created.
18690                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18691                // Collected, because ending the scope also closes the co-tenant
18692                // route, whose GOODBYE comes first.
18693                let mut goodbyes = Vec::new();
18694                while let Ok(outbound) = plexus_rx.try_recv() {
18695                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18696                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18697                }
18698                assert!(
18699                    goodbyes.contains(&(bind_channel, bind_epoch)),
18700                    "{goodbyes:?}"
18701                );
18702                assert!(rig
18703                    .handler
18704                    .registry
18705                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18706                    .unwrap()
18707                    .is_some());
18708            }
18709            assert_eq!(ended, Some(true));
18710            // The co-tenant stayed up through the change, and closed only when
18711            // the scope ended, by the drain rule rather than by the commit.
18712            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18713        }
18714
18715        /// Each row of the drain table on one set of routes: the owner's, a
18716        /// bare carrier's, and a targeted carrier's to each of its targets.
18717        #[tokio::test]
18718        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18719            struct Case {
18720                name: &'static str,
18721                change: fn(&mut ScopeRecord),
18722                /// Closed routes by index: owner->plexus, aft->plexus,
18723                /// broca->plexus, broca->other.
18724                closed: [Option<RouteCloseReason>; 4],
18725            }
18726            use RouteCloseReason::*;
18727            let cases = [
18728                Case {
18729                    name: "a carrier entry removed",
18730                    change: |r| {
18731                        r.carriers.retain(|c| {
18732                            c.principal
18733                                != Principal::Reserved {
18734                                    module_id: AFT.to_string(),
18735                                }
18736                        })
18737                    },
18738                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18739                },
18740                Case {
18741                    name: "a target removed from a carrier",
18742                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18743                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
18744                },
18745                Case {
18746                    name: "a bare carrier narrowed to targets",
18747                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18748                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18749                },
18750                Case {
18751                    name: "delegates turned off",
18752                    change: |r| r.attributes.delegates = false,
18753                    closed: [Some(ScopeDelegationChanged); 4],
18754                },
18755                Case {
18756                    name: "agent_id changed",
18757                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18758                    closed: [Some(ScopeDelegationChanged); 4],
18759                },
18760                Case {
18761                    name: "a carrier added, child owners changed, the record re-sent",
18762                    change: |r| {
18763                        r.carriers.push(carrier(MAGIC, None));
18764                        r.child_owners.push(Principal::Reserved {
18765                            module_id: MAGIC.to_string(),
18766                        });
18767                    },
18768                    closed: [None; 4],
18769                },
18770                Case {
18771                    name: "a target added",
18772                    change: |r| {
18773                        r.carriers[1]
18774                            .targets
18775                            .as_mut()
18776                            .unwrap()
18777                            .push("third".to_string())
18778                    },
18779                    closed: [None; 4],
18780                },
18781                Case {
18782                    name: "delegates turned on",
18783                    change: |r| r.attributes.delegates = true,
18784                    closed: [None; 4],
18785                },
18786            ];
18787            for case in cases {
18788                let mut rig = rig().await;
18789                rig.sync(vec![session(1)]).await;
18790                let scope = || Some(rig_selector("s", Some(1)));
18791                let mut routes = [
18792                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
18793                    rig.bound(Some(AFT), PLEXUS, scope()).await,
18794                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
18795                    rig.bound(Some(BROCA), OTHER, scope()).await,
18796                ];
18797                let mut record = session(1);
18798                (case.change)(&mut record);
18799                rig.sync(vec![record]).await;
18800                for (index, expected) in case.closed.iter().enumerate() {
18801                    let route = &mut routes[index];
18802                    match expected {
18803                        Some(reason) => {
18804                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
18805                            assert_eq!(
18806                                route.closed_reason(),
18807                                *reason,
18808                                "{}: route {index}",
18809                                case.name
18810                            );
18811                        }
18812                        None => {
18813                            assert!(rig.live(route), "{}: route {index} closed", case.name);
18814                            assert!(
18815                                route.untouched(),
18816                                "{}: route {index} was told something",
18817                                case.name
18818                            );
18819                        }
18820                    }
18821                }
18822            }
18823        }
18824
18825        #[tokio::test]
18826        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18827            // Removed, and replaced by a higher epoch.
18828            for next in [Vec::new(), vec![session(2)]] {
18829                let mut rig = rig().await;
18830                rig.sync(vec![session(1)]).await;
18831                let mut route = rig
18832                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18833                    .await;
18834                rig.sync(next).await;
18835                assert!(!rig.live(&route));
18836                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18837            }
18838
18839            // A child whose parent ends: its routes close as parent-ended, the
18840            // child stays live, and routes under the parent close as ended.
18841            let mut rig = rig().await;
18842            let mut child = session(1);
18843            child.scope_ref = "child".to_string();
18844            child.kind = ScopeKind::Worker;
18845            child.parent = Some(ScopeParent::new(
18846                Principal::Reserved {
18847                    module_id: OWNER.to_string(),
18848                },
18849                "s".to_string(),
18850                1,
18851            ));
18852            rig.sync(vec![session(1), child.clone()]).await;
18853            let mut child_route = rig
18854                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18855                .await;
18856            assert_eq!(
18857                child_route.stamp().unwrap().parent_state,
18858                Some(ParentState::Linked)
18859            );
18860            rig.sync(vec![child]).await;
18861            assert!(!rig.live(&child_route));
18862            assert_eq!(
18863                child_route.closed_reason(),
18864                RouteCloseReason::ScopeParentEnded
18865            );
18866        }
18867
18868        #[tokio::test]
18869        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18870        ) {
18871            let mut rig = rig().await;
18872            rig.sync(vec![session(1)]).await;
18873            let mut route = rig
18874                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18875                .await;
18876            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18877            rig.sync(vec![session(1)]).await;
18878            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18879            assert!(rig.live(&route) && route.untouched());
18880
18881            // A call in flight on the route when another carrier is added. A
18882            // forwarded REQUEST holds one credit on the route's flow until the
18883            // module answers; the router takes it exactly like this.
18884            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18885                .forwarding
18886                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18887                .unwrap()
18888            else {
18889                panic!("the route is bound");
18890            };
18891            binding.flow.acquire_tagged(9, false).await.unwrap();
18892            let mut widened = session(1);
18893            widened.carriers.push(carrier(MAGIC, None));
18894            rig.sync(vec![widened]).await;
18895            assert!(rig.live(&route) && route.untouched());
18896            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18897            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18898            // The call's credit is still held on an open flow, so its answer
18899            // will be delivered: closing the route would have closed the flow.
18900            assert_eq!(binding.flow.in_flight(), 1);
18901            binding
18902                .flow
18903                .acquire_tagged(10, false)
18904                .await
18905                .expect("the flow is still open");
18906        }
18907
18908        /// A swap's superseded endpoint keeps its routes until drained; ending
18909        /// the scope closes them there too.
18910        #[tokio::test]
18911        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18912            let mut rig = rig().await;
18913            rig.sync(vec![session(1)]).await;
18914            let mut on_incumbent = rig
18915                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18916                .await;
18917
18918            // Swap plexus: register a candidate and cut over, leaving the
18919            // incumbent superseded with the route still on it.
18920            let (candidate, _candidate_rx) = wide_ctx(9);
18921            let registration = rig
18922                .handler
18923                .registry
18924                .register_candidate_with_control_ops(
18925                    manifest(PLEXUS, PROTOCOL_VERSION),
18926                    PROTOCOL_VERSION,
18927                    candidate.connection_id,
18928                    module_baseline_control_ops(),
18929                )
18930                .unwrap();
18931            rig.forwarding
18932                .register_candidate_module_connection(
18933                    candidate.connection_id,
18934                    PLEXUS.to_string(),
18935                    PROTOCOL_VERSION,
18936                    manifest_concurrency(&registration.manifest),
18937                    candidate.egress.clone(),
18938                )
18939                .unwrap();
18940            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18941            rig.handler
18942                .registry
18943                .promote_candidate(PLEXUS)
18944                .unwrap()
18945                .unwrap();
18946            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18947
18948            rig.sync(Vec::new()).await;
18949            assert!(!rig.live(&on_incumbent));
18950            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18951            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18952            let goodbye = incumbent_rx
18953                .try_recv()
18954                .expect("the superseded endpoint is told")
18955                .frame;
18956            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18957        }
18958    }
18959}
18960
18961#[cfg(test)]
18962mod concurrency_default_exposure_tests {
18963    use super::*;
18964
18965    fn hello_body(role_json: &str) -> Vec<u8> {
18966        format!(
18967            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18968        )
18969        .into_bytes()
18970    }
18971
18972    fn manifest_from(body: &[u8]) -> ModuleManifest {
18973        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18974        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18975            .expect("manifest parses")
18976    }
18977
18978    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18979
18980    #[test]
18981    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18982        let body = hello_body(&format!(
18983            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18984        ));
18985        let manifest = manifest_from(&body);
18986        // Precondition: serde really resolved it to the default, so the typed
18987        // manifest alone cannot answer the question this probe exists for.
18988        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18989        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18990    }
18991
18992    #[test]
18993    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18994        let body = hello_body(&format!(
18995            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18996        ));
18997        let manifest = manifest_from(&body);
18998        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18999        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19000    }
19001
19002    #[test]
19003    fn non_management_roles_are_never_reported() {
19004        let body = hello_body(
19005            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19006        );
19007        let manifest = manifest_from(&body);
19008        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19009    }
19010}