Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_confirm_operations,
25        validate_hello_event_declarations, validate_hello_flow_operations,
26        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29    },
30    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
31    scope::{
32        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
33        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
34        SCOPE_SYNC_OP,
35    },
36    session::{
37        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
38        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
39        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
40        ROLE_VERSIONS_FIELD,
41    },
42    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
43    Priority, RouteTarget, PROTOCOL_VERSION,
44};
45use tokio::time::{timeout_at, Instant};
46use tracing::{debug, info, instrument::WithSubscriber, warn};
47
48use crate::{
49    caller_identity::CallerIdentity,
50    capability_requirements::{
51        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
52        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
53        RequirementStatus, RuntimeModule,
54    },
55    child_roster::DaemonShutdownFlag,
56    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
57    forwarding::{
58        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
59        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
60        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
61    },
62    observability::{
63        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
64    },
65    privileged_prompt::{prompt_text, PrivilegedOperation},
66    provenance::{
67        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
68    },
69    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
70    router::{daemon_error_flags, RouteCtx, RouterError},
71    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
72    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
73    stderr_tail::{CaptureState, TailEntry},
74    supervise::{
75        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
76        SupervisorHandle, SwapHelloAdmission,
77    },
78    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
79};
80
81/// Lowest envelope version this subc build will negotiate.
82///
83/// Module HELLO negotiation is exact: peers must use the daemon's locked
84/// protocol version. Older and newer peers receive `version_unsupported` and
85/// are not registered.
86pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
87
88const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
89const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
90const CAP_PING_PONG: &str = "ping_pong_v1";
91const CAP_SESSION_ATTACH: &str = "session_attach_v1";
92const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
93
94const SUBC_CONTROL_OPS: &[&str] = &[
95    ops::DAEMON_SHUTDOWN_INTENT,
96    ops::ACTIVITY_RECORD,
97    ops::ACTIVITY_QUERY,
98    ops::ACTIVITY_SUBSCRIBE,
99    ops::SERVER_DESCRIBE,
100    ops::CATALOG_LIST,
101    ops::ROUTE_OPEN,
102    ops::ROUTE_POLL,
103    ops::ROUTE_CLOSING,
104    ops::ROUTE_CLOSED,
105    ops::SUPERVISOR_LIST,
106    ops::SUPERVISOR_RESTART,
107    ops::SUPERVISOR_SWAP,
108    ops::SUPERVISOR_RELOAD,
109    ops::SUPERVISOR_RESCAN,
110    ops::SUPERVISOR_RELEASE_RESERVED,
111    ops::SUPERVISOR_SET_ENABLED,
112    ops::SUPERVISOR_HEALTH_PROBE,
113    ops::SUPERVISOR_HEALTH,
114    ops::SUPERVISOR_STDERR_TAIL,
115    ops::SUPERVISOR_TERMINALS,
116    ops::SUPERVISOR_ROUTES,
117    ops::SUPERVISOR_PROVENANCE,
118    ops::SUPERVISOR_SPAWN_SNAPSHOT,
119    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
120];
121
122/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
123/// announced by a capability in `server.describe` instead, and clients check
124/// that capability before sending it. `ck daemon --json` prints `subc_ops`
125/// verbatim and scripts read that output, so a new name there would change it.
126/// A malformed body for one of these is still refused `invalid_control_body`
127/// rather than as an unknown op.
128const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
129
130mod rescan_checked;
131use rescan_checked::{RescanMode, RescanPlan};
132
133const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
134    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
135    "supervisor.live_roots",
136    SCOPE_SYNC_OP,
137    SCOPE_APPLY_OP,
138    SCOPE_DESCRIBE_OP,
139    "operator.confirm",
140];
141
142/// Module-originated ops the daemon answers but does not advertise in
143/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
144/// belongs to is incomplete, so no module is told it works before it does.
145const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
146
147const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
148
149/// How long subc waits for a module to ack a relayed route.bind before returning
150/// `module_timeout`. The ack waits on the module's own configure, which for AFT
151/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
152/// and DB-open work — on a cold page cache or a large repo that legitimately
153/// exceeds a couple of seconds. The default is generous because rejecting a VALID
154/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
155/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
156pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
157
158/// How many CONSECUTIVE full-budget relay timeouts against one target module
159/// open that module's bind-relay breaker.
160///
161/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
162/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
163/// so three consecutive full-budget timeouts take ~36s to observe: every client
164/// whose open contributed to opening the breaker had already given up on its
165/// own. That is what makes opening the breaker unable to turn a call that would
166/// have succeeded into a refusal — it can only make an already-failing module
167/// fail faster.
168///
169/// Two would be reachable inside one default deadline. One would convict a
170/// module on a single cold-cache bind, which is exactly the valid-but-slow case
171/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
172pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
173
174/// How long a module's bind-relay breaker stays open before exactly one
175/// `route.open` is let through as a probe.
176///
177/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
178/// re-pays a full-budget stall almost continuously, and the breaker stops being
179/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
180/// deadline: a client that starts retrying after the module recovers has to get
181/// a probe opportunity inside its own deadline, or the breaker converts a
182/// recovered module into a failed call — the failure it exists to prevent,
183/// pointed the other way.
184///
185/// 20s sits between those with room on both sides, and it caps what a wedged
186/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
187/// rather than one per `route.open` per connection. The stall that motivated
188/// this, with its measurements, is written up in
189/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
190/// waited the whole budget out.
191pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
192
193const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
194const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
195
196fn reload_verdict(
197    configured: &Path,
198    spawned_from: Option<&Path>,
199    image: subc_control::RunningImageAgreement,
200) -> PendingReloadVerdict {
201    let path = match spawned_from {
202        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
203            ReloadPathAgreement::Match
204        }
205        Some(spawned_from) => ReloadPathAgreement::Mismatch {
206            configured: configured.to_path_buf(),
207            spawned_from: spawned_from.to_path_buf(),
208        },
209        None => ReloadPathAgreement::Unavailable {
210            reason: if matches!(
211                image,
212                subc_control::RunningImageAgreement::Unavailable {
213                    reason: subc_control::RunningImageUnavailableReason::NotRunning
214                }
215            ) {
216                ReloadPathUnavailableReason::NotRunning
217            } else {
218                ReloadPathUnavailableReason::SpawnedPathUnavailable
219            },
220        },
221    };
222    PendingReloadVerdict {
223        path,
224        image,
225        spec: None,
226    }
227}
228
229fn launch_spec_agreement(
230    configured: &crate::supervise::ModuleSpec,
231    running: bool,
232    spawned: Option<&crate::supervise::LaunchSpecDigest>,
233) -> subc_control::LaunchSpecAgreement {
234    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
235    if !running {
236        return Agreement::Unavailable {
237            reason: Reason::NotRunning,
238        };
239    }
240    match spawned {
241        None => Agreement::Unavailable {
242            reason: Reason::SpawnedSpecUnavailable,
243        },
244        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
245            Agreement::Match
246        }
247        Some(_) => Agreement::Mismatch,
248    }
249}
250
251fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
252    if configured == spawned_from {
253        return true;
254    }
255    #[cfg(windows)]
256    {
257        // Windows spawn records the resolved verbatim path (\\?\...), while
258        // configuration keeps its original spelling. Resolve both with the OS
259        // instead of stripping prefixes or folding case, which could conflate
260        // distinct names in case-sensitive directories. Different resolved
261        // paths still differ even if their files have identical content.
262        match (
263            std::fs::canonicalize(configured),
264            std::fs::canonicalize(spawned_from),
265        ) {
266            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
267            _ => false,
268        }
269    }
270    #[cfg(not(windows))]
271    {
272        false
273    }
274}
275
276#[derive(Clone)]
277struct DaemonProvenanceFacts {
278    build: DaemonBuildProvenance,
279    pid: Option<u32>,
280    started_at_ms: Option<u64>,
281    start_clock: Option<crate::clock::StartClock>,
282    executable_path: Option<PathBuf>,
283    executable_identity: Option<SpawnedFileIdentity>,
284    process_start_time: Option<u64>,
285    probe: ExecutableIdentityProbe,
286}
287
288impl Default for DaemonProvenanceFacts {
289    fn default() -> Self {
290        Self {
291            build: DaemonBuildProvenance {
292                build_git_sha: None,
293                build_lock_digest: None,
294            },
295            pid: None,
296            started_at_ms: None,
297            start_clock: None,
298            executable_path: None,
299            executable_identity: None,
300            process_start_time: None,
301            probe: ExecutableIdentityProbe::default(),
302        }
303    }
304}
305
306#[derive(Debug, Clone)]
307struct SupervisorRescanContext {
308    supervisor: Supervisor,
309    config_path: PathBuf,
310    configured_port: Option<u16>,
311    storage_config: Option<crate::daemon_config::StorageConfig>,
312    admission_facts_carrier_module_id: Option<String>,
313    admission_facts_targets: Option<Vec<String>>,
314    scope_authority_owners: Vec<String>,
315    privileged_operator_prompt: PrivilegedOperatorPrompt,
316    /// Counts executed rescans that set out to change the supervised modules'
317    /// staged or applied configuration, counted before the first change so a
318    /// rescan that fails partway still counts. With the daemon incarnation it
319    /// names the state a `supervisor.rescan_checked` preview was computed
320    /// against.
321    generation: Arc<std::sync::atomic::AtomicU64>,
322}
323
324/// Refusal labels passed to `observe_route_open_refusal` that mean the target
325/// module is not serving right now, and so open or extend an outage in the
326/// route outage tracker. Every one of them is only reachable after the target
327/// was found in the registry, which is what keeps an arbitrary client-chosen
328/// id from ever creating tracker state.
329///
330/// Deliberately absent: `not_registered` and `removed` (the id may be
331/// anything a client sent, and a removed module is gone on purpose),
332/// `protocol_none` (such a module never serves routes, so nothing is out),
333/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
334/// capability and admission-facts refusals (they refuse the caller, not a
335/// module outage), and `relay_reservation_failed` (its code ranges over
336/// capacity limits as well as a vanished connection). Capacity, breaker,
337/// relay-timeout and module-rejection refusals do not pass through that
338/// function at all; the breaker logs its own transitions.
339///
340/// The two not-serving refusals that bypass that function record themselves
341/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
342/// `required_capability_unprovided` is not tracked: the module itself is up,
343/// and the outage belongs to the missing provider.
344const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
345    "reloading",
346    "supervisor_not_live",
347    "registration_not_active",
348    "no_forwarding_connection",
349    "relay_send_failed",
350];
351
352/// Real channel-0 control handler for subc itself.
353#[derive(Clone)]
354pub struct ControlHandler {
355    registry: Arc<Registry>,
356    activity: Option<crate::activity::ActivityLedger>,
357    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
358    forwarding: Arc<ForwardingTable>,
359    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
360    supervisor: SupervisorHandle,
361    subc_capabilities: Arc<[String]>,
362    /// Daemon-wide route.bind relay budget. Used as the fallback when the
363    /// target module has no per-module override in
364    /// `route_bind_relay_timeouts`.
365    route_bind_relay_timeout: Duration,
366    /// Per-module route.bind relay budget overrides, keyed by module id. When
367    /// `handle_route_open` resolves the deadline for a target module, a
368    /// per-module entry wins over the daemon-wide value above.
369    route_bind_relay_timeouts: BTreeMap<String, Duration>,
370    /// Per-target-module bind-relay breaker state. Shared with the forwarding
371    /// table, which is where a new module connection resets it.
372    route_bind_breakers: RouteBindBreakers,
373    /// Live relay admissions keyed by target module. Shared through the
374    /// forwarding table so cloned or separately built handlers enforce one cap.
375    route_bind_concurrency: RouteBindConcurrency,
376    /// Start and end of each module's not-serving period as seen by
377    /// `route.open`, so an outage gets one line at each edge instead of only
378    /// the per-refusal INFO lines. Taken from the forwarding table, so every
379    /// handler built over one table shares it.
380    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
381    /// Consecutive relay timeouts that open a module's breaker.
382    route_bind_breaker_threshold: u32,
383    /// How long a breaker stays open before one probe is admitted.
384    route_bind_breaker_cooldown: Duration,
385    health_probe_timeout: Duration,
386    /// Central storage policy. When set, each registering module receives its
387    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
388    storage_config: Option<crate::daemon_config::StorageConfig>,
389    /// The machine id established at boot, served on every HELLO_ACK and on
390    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
391    /// changes the file, never this value. `None` serves no id.
392    machine_id: Option<crate::machine_id::MachineId>,
393    admission_facts_carrier_module_id: Option<String>,
394    admission_facts_targets: Option<Vec<String>>,
395    /// Scope records with their sync authorities and tombstones; see
396    /// `crate::scopes`. Shared by clones of this handler, so every connection
397    /// reads and writes one table.
398    scopes: Arc<RwLock<ScopeTable>>,
399    /// Wakes the expiry loop when an accepted scope change may have added a
400    /// deadline, so the loop can wait without ticking while no live scope has
401    /// one. A permit is stored when nothing waits, so no wake is lost.
402    scope_deadline_added: Arc<tokio::sync::Notify>,
403    /// The configured `scope_authority_owners`, kept so a rescan can report a
404    /// changed value as needing a daemon restart; rescan never applies it.
405    scope_authority_owners: Vec<String>,
406    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
407    privileged_operator_prompt: PrivilegedOperatorPrompt,
408    /// The local and peer address of each accepted TCP client connection, read
409    /// from the daemon's own socket, so a privileged request can name its caller.
410    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
411    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
412    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
413    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
414    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
415    pub(crate) operator_apps:
416        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417    /// One launch entry per app, behind a leaf lock: no other lock is taken
418    /// while it is held. Entries are memory-only, so a daemon or
419    /// installed-service restart drops them and a relaunched daemon never
420    /// mistakes an older process for its own launch.
421    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
422    /// Serializes `app.open` launches: the liveness check in the decision and
423    /// the later publish must not interleave across concurrent calls, or two
424    /// calls could both launch and the second publish would orphan the first
425    /// process. Refusals and `already_running` answers share the same lock;
426    /// `app.open` is rare, so the cost is negligible.
427    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
428    // Portable wire samples exercise dispatch without starting an OS process.
429    #[cfg(test)]
430    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
431    #[cfg(test)]
432    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
433    /// Test-only Windows signature and flash hooks. Production builds carry
434    /// neither: with the publisher unset, the production table never reaches
435    /// verification, and the default attention hook only logs.
436    #[allow(dead_code)]
437    #[cfg(windows)]
438    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
439    #[allow(dead_code)]
440    #[cfg(windows)]
441    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
442    /// Test-only tracking fault hook: makes opening the tracking handle on
443    /// the launched pid fail, so a test can prove an untrackable launch is
444    /// stopped rather than left running.
445    #[allow(dead_code)]
446    #[cfg(windows)]
447    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
448    /// Test-only count of Windows spawn attempts, so a test can prove a
449    /// refused launch started no process. Unit tests only.
450    #[cfg(all(test, windows))]
451    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
452    #[cfg(test)]
453    caller_lookup: Option<TestCallerLookup>,
454    /// The launch nonce each module connection presented at HELLO, which is how
455    /// a `scope.sync` is matched to the owner's current launch.
456    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
457    rescan: Option<SupervisorRescanContext>,
458    connected_clients: ConnectedClients,
459    counters: DaemonCounters,
460    /// Why this daemon serves no local endpoint, when binding one failed. `None`
461    /// when the endpoint is served, and also on a handler that was never told.
462    local_endpoint_unavailable: Option<String>,
463    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
464    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
465    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
466    /// their usual severity (see [`log_requirement_events`]). A handler built
467    /// without one has a flag that is never set.
468    daemon_shutdown: DaemonShutdownFlag,
469    daemon_provenance: DaemonProvenanceFacts,
470    #[cfg(test)]
471    control_dispatch_delay: Option<Duration>,
472    #[cfg(test)]
473    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
474    /// Holds `route.open` after the route.bind is queued to the module and
475    /// before it is marked enqueued, so a test can make the module's reply (or
476    /// a client teardown) land exactly inside that window.
477    #[cfg(test)]
478    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
479    /// Holds a HELLO after its registry entry is reserved and before its
480    /// forwarding endpoint is installed, so a test can read the catalog and
481    /// open routes while the module is registered but not yet routable. A
482    /// blocking barrier because the HELLO handler is synchronous; the test
483    /// runs the HELLO on its own thread.
484    #[cfg(test)]
485    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
486}
487
488impl fmt::Debug for ControlHandler {
489    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
490        f.debug_struct("ControlHandler")
491            .field("registry", &self.registry)
492            .field("forwarding", &self.forwarding)
493            .field("process_liveness", &self.process_liveness.is_some())
494            .field("supervisor", &self.supervisor)
495            .field("subc_capabilities", &self.subc_capabilities)
496            .finish()
497    }
498}
499
500struct RouteOpenRequest {
501    target: RouteTarget,
502    identity: BindIdentity,
503    consumer_identity: Option<ConsumerIdentity>,
504    consumer_capabilities: Option<Vec<String>>,
505    role_versions: Option<BTreeMap<String, String>>,
506    busy_refusals: Option<bool>,
507    admission_facts: Option<serde_json::Value>,
508    scope: Option<ScopeSelector>,
509}
510
511#[derive(Clone, Copy)]
512enum RouteOpenPhase {
513    Admission,
514    ProjectRoot,
515    EgressReservation,
516    BindSend,
517    BindAckWait,
518}
519
520/// Store elapsed time for each route-open phase, including attempts that end
521/// in an early refusal or backend error. Admission also includes rechecks and
522/// commit bookkeeping performed between the other timed phases.
523struct RouteOpenTimings {
524    module_id: String,
525    principal: String,
526    admission: Duration,
527    project_root: Duration,
528    egress_reservation: Duration,
529    bind_send: Duration,
530    bind_ack_wait: Duration,
531    phase: Option<RouteOpenPhase>,
532    phase_started_at: Instant,
533}
534
535impl RouteOpenTimings {
536    fn new() -> Self {
537        Self {
538            module_id: String::new(),
539            // Refusals before attestation must not claim a verified principal.
540            principal: "unresolved".to_string(),
541            admission: Duration::ZERO,
542            project_root: Duration::ZERO,
543            egress_reservation: Duration::ZERO,
544            bind_send: Duration::ZERO,
545            bind_ack_wait: Duration::ZERO,
546            phase: Some(RouteOpenPhase::Admission),
547            phase_started_at: Instant::now(),
548        }
549    }
550
551    fn finish(&mut self) {
552        let elapsed = self.phase_started_at.elapsed();
553        match self.phase.take() {
554            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
555            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
556            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
557            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
558            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
559            None => {}
560        }
561    }
562
563    fn enter(&mut self, phase: RouteOpenPhase) {
564        self.finish();
565        self.phase = Some(phase);
566        self.phase_started_at = Instant::now();
567    }
568}
569
570struct RouteBindReservationGuard {
571    forwarding: Arc<ForwardingTable>,
572    endpoint: ModuleEndpointId,
573    relay_corr: u64,
574    armed: bool,
575}
576
577struct ModuleControlRpcGuard {
578    forwarding: Arc<ForwardingTable>,
579    endpoint: ModuleEndpointId,
580    corr: u64,
581    armed: bool,
582}
583
584impl ModuleControlRpcGuard {
585    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
586        Self {
587            forwarding,
588            endpoint,
589            corr,
590            armed: true,
591        }
592    }
593
594    fn disarm(&mut self) {
595        self.armed = false;
596    }
597}
598
599impl Drop for ModuleControlRpcGuard {
600    fn drop(&mut self) {
601        if self.armed {
602            let _ = self
603                .forwarding
604                .cancel_module_control_rpc(self.endpoint, self.corr);
605        }
606    }
607}
608
609impl RouteBindReservationGuard {
610    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
611        Self {
612            forwarding,
613            endpoint,
614            relay_corr,
615            armed: true,
616        }
617    }
618
619    fn release_and_disarm(&mut self) {
620        if !self.armed {
621            return;
622        }
623        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
624            self.endpoint,
625            self.relay_corr,
626            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
627        ) {
628            send_goodbye_target_best_effort(
629                &self.forwarding.counters(),
630                &target,
631                "canceled route.bind",
632            );
633        }
634        self.armed = false;
635    }
636
637    fn disarm(&mut self) {
638        self.armed = false;
639    }
640}
641
642impl Drop for RouteBindReservationGuard {
643    fn drop(&mut self) {
644        self.release_and_disarm();
645    }
646}
647
648/// Per-target-module circuit breaker around the `route.bind` relay.
649///
650/// The connection reader is serial per connection, so a module whose `on_bind`
651/// sits on the ack blocks every LATER frame on the connections that call it,
652/// including calls to unrelated modules. This does not make any module's bind
653/// fast; it stops the daemon paying the full budget again and again for a
654/// condition it has already observed.
655///
656/// State is keyed by TARGET MODULE and shared by every connection: a wedged
657/// module wedges everyone, so what one connection learned should protect the
658/// rest.
659///
660/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
661/// relay to that module has actually timed out, and is removed again when a
662/// relay is accepted or the module reconnects, so it cannot grow with traffic
663/// or with modules that behave.
664///
665/// # Why a `std` mutex here is not the head-of-line defect again
666///
667/// Acquisition never awaits. The critical section is a hash lookup plus a few
668/// integer updates, with no I/O and no `.await` inside it, so a reader task
669/// cannot be descheduled behind it the way it can behind
670/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
671/// primitive, held for the same kind of work, as the refusal counter this very
672/// path already increments.
673///
674/// It is also NOT on the data-plane splice path: only `route.open` and module
675/// registration touch it, so bound-route frames gain no state check and no
676/// contention.
677#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindBreakers {
679    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
680}
681
682#[derive(Debug, Clone, Default)]
683pub(crate) struct RouteBindConcurrency {
684    modules: Arc<Mutex<HashMap<String, usize>>>,
685}
686
687struct RouteBindConcurrencyGuard {
688    concurrency: RouteBindConcurrency,
689    module_id: String,
690}
691
692impl RouteBindConcurrency {
693    /// Admit without waiting. Waiting here would move the bind stall from the
694    /// module reply to a semaphore and restore reader head-of-line blocking.
695    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
696        let mut modules = self
697            .modules
698            .lock()
699            .expect("route.bind concurrency mutex poisoned");
700        let in_flight = modules.entry(module_id.to_string()).or_default();
701        if *in_flight >= limit {
702            return Err(*in_flight);
703        }
704        *in_flight += 1;
705        Ok(RouteBindConcurrencyGuard {
706            concurrency: self.clone(),
707            module_id: module_id.to_string(),
708        })
709    }
710}
711
712impl Drop for RouteBindConcurrencyGuard {
713    fn drop(&mut self) {
714        let mut modules = self
715            .concurrency
716            .modules
717            .lock()
718            .expect("route.bind concurrency mutex poisoned");
719        let remove = {
720            let in_flight = modules
721                .get_mut(&self.module_id)
722                .expect("admitted route.bind has a concurrency entry");
723            *in_flight -= 1;
724            *in_flight == 0
725        };
726        if remove {
727            modules.remove(&self.module_id);
728        }
729    }
730}
731
732#[derive(Debug, Default)]
733struct ModuleBreakerState {
734    /// Relay timeouts observed with no accepted relay in between.
735    consecutive_timeouts: u32,
736    /// `Some` while the breaker is open: the instant the cooldown expires and
737    /// the next arrival may probe. `None` means closed.
738    cooldown_until: Option<Instant>,
739    /// A half-open probe has been admitted and has not settled yet. This is
740    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
741    /// that read the cooldown, so concurrent opens arriving at the moment the
742    /// cooldown expires cannot all decide that they are the probe.
743    probe_in_flight: Option<Arc<()>>,
744}
745
746/// What the breaker decided for one `route.open`, before any relay work.
747enum RouteBindAdmission<'a> {
748    Admitted {
749        guard: RouteBindBreakerGuard<'a>,
750        /// This open is the single half-open probe, so the transition is worth
751        /// one log line.
752        probe: bool,
753    },
754    Refused {
755        consecutive_timeouts: u32,
756        /// What is left of the cooldown. Zero when the refusal is because the
757        /// one probe is already in flight rather than because the cooldown has
758        /// not elapsed.
759        retry_in: Duration,
760        probe_in_flight: bool,
761    },
762}
763
764/// An outstanding admission, which must be told how its relay settled.
765///
766/// `Drop` settles it as inconclusive, so an early return between admission and
767/// the relay -- or the whole handler being cancelled when the client
768/// disconnects -- releases a half-open probe slot instead of leaving the
769/// breaker wedged half-open with no further probes.
770struct RouteBindBreakerGuard<'a> {
771    breakers: RouteBindBreakers,
772    module_id: &'a str,
773    probe_token: Option<Arc<()>>,
774    settled: bool,
775}
776
777impl RouteBindBreakerGuard<'_> {
778    /// The module answered within the budget and took the bind. THE ONLY
779    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
780    /// breaker, which is a transition worth logging.
781    fn record_accepted(&mut self) -> bool {
782        self.settled = true;
783        self.breakers.record_accepted(self.module_id)
784    }
785
786    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
787    /// COUNTS TOWARD OPENING.
788    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
789        self.settled = true;
790        self.breakers.record_timeout(
791            self.module_id,
792            self.probe_token.as_ref(),
793            threshold,
794            cooldown,
795        )
796    }
797
798    /// Everything else: the module REJECTED the bind, its connection went away
799    /// mid-relay, or the waiter was cancelled.
800    ///
801    /// None of these is evidence that a module is slow, and each already has
802    /// its own refusal with its own code. A module that rejects a bind in
803    /// microseconds is healthy and must never be convicted for it; a module
804    /// that died has said nothing about the module that replaces it. So these
805    /// neither increment nor reset the count -- they only release a probe slot.
806    fn record_inconclusive(&mut self) {
807        self.settled = true;
808        self.breakers
809            .record_inconclusive(self.module_id, self.probe_token.as_ref());
810    }
811}
812
813impl Drop for RouteBindBreakerGuard<'_> {
814    fn drop(&mut self) {
815        if !self.settled {
816            self.breakers
817                .record_inconclusive(self.module_id, self.probe_token.as_ref());
818        }
819    }
820}
821
822/// The breaker moved to open, reported so the caller can log it outside the
823/// lock. Opening is rare and load-bearing; the refusals that follow are
824/// frequent and are counted rather than logged.
825struct BreakerOpened {
826    consecutive_timeouts: u32,
827    /// True when a failed probe re-opened an already-open breaker, which reads
828    /// very differently in a log from a first opening.
829    reopened_after_probe: bool,
830}
831
832impl RouteBindBreakers {
833    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
834        self.modules
835            .lock()
836            .expect("route.bind breaker mutex poisoned")
837    }
838
839    /// Decide whether this `route.open` may attempt its relay. Takes the map
840    /// lock and nothing else, and never awaits.
841    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
842        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
843            probe: probe_token.is_some(),
844            guard: RouteBindBreakerGuard {
845                breakers: self.clone(),
846                module_id,
847                probe_token,
848                settled: false,
849            },
850        };
851
852        let mut modules = self.lock();
853        let Some(state) = modules.get_mut(module_id) else {
854            return admitted(None);
855        };
856        let Some(cooldown_until) = state.cooldown_until else {
857            return admitted(None);
858        };
859        if state.probe_in_flight.is_some() {
860            return RouteBindAdmission::Refused {
861                consecutive_timeouts: state.consecutive_timeouts,
862                retry_in: Duration::ZERO,
863                probe_in_flight: true,
864            };
865        }
866        let now = Instant::now();
867        if now < cooldown_until {
868            return RouteBindAdmission::Refused {
869                consecutive_timeouts: state.consecutive_timeouts,
870                retry_in: cooldown_until - now,
871                probe_in_flight: false,
872            };
873        }
874        let token = Arc::new(());
875        state.probe_in_flight = Some(Arc::clone(&token));
876        admitted(Some(token))
877    }
878
879    fn record_accepted(&self, module_id: &str) -> bool {
880        self.lock()
881            .remove(module_id)
882            .is_some_and(|state| state.cooldown_until.is_some())
883    }
884
885    fn record_timeout(
886        &self,
887        module_id: &str,
888        probe_token: Option<&Arc<()>>,
889        threshold: u32,
890        cooldown: Duration,
891    ) -> Option<BreakerOpened> {
892        let mut modules = self.lock();
893        let state = modules.entry(module_id.to_string()).or_default();
894        let was_open = state.cooldown_until.is_some();
895        let was_probe = Self::owns_probe(state, probe_token);
896        if was_probe {
897            state.probe_in_flight = None;
898        }
899        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
900        if state.consecutive_timeouts < threshold {
901            return None;
902        }
903        state.cooldown_until = Some(Instant::now() + cooldown);
904        Some(BreakerOpened {
905            consecutive_timeouts: state.consecutive_timeouts,
906            reopened_after_probe: was_open && was_probe,
907        })
908    }
909
910    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
911        // A relay can finish after the breaker was reset or after it opened
912        // again and started a new probe. Only the guard whose token matches the
913        // active probe may release it, so a late relay never frees a newer probe.
914        state
915            .probe_in_flight
916            .as_ref()
917            .zip(token)
918            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
919    }
920
921    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
922        if let Some(state) = self.lock().get_mut(module_id) {
923            if Self::owns_probe(state, probe_token) {
924                state.probe_in_flight = None;
925            }
926        }
927    }
928
929    /// Discard what was learned about a module, because the process it was
930    /// learned about is gone. Returns the discarded count when it was non-zero.
931    ///
932    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
933    /// `module_id` is a configuration identity that outlives any particular
934    /// child; what the breaker observed was the process behind the module
935    /// connection of the moment. When a new connection registers under that id
936    /// the verdict's subject no longer exists, so the verdict is stale by
937    /// construction rather than merely likely to be wrong. Keeping it would
938    /// apply a dead process's record to a live one, which is the same defect
939    /// class this breaker exists to stop the daemon committing.
940    ///
941    /// A half-open probe in flight is discarded with the rest: it was a
942    /// question about the old process.
943    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
944        self.lock()
945            .remove(module_id)
946            .map(|state| state.consecutive_timeouts)
947            .filter(|discarded| *discarded > 0)
948    }
949
950    /// Open breakers, for the `server.describe` counters object. `None` when
951    /// none is open, so the key stays absent rather than present-and-empty.
952    ///
953    /// This is the operator's answer to "is this module refusing instantly or
954    /// is it fine?", which look identical from a client that retries and then
955    /// succeeds.
956    fn open_snapshot(&self) -> Option<serde_json::Value> {
957        let now = Instant::now();
958        let modules = self.lock();
959        let open = modules
960            .iter()
961            .filter_map(|(module_id, state)| {
962                let cooldown_until = state.cooldown_until?;
963                Some((
964                    module_id.clone(),
965                    serde_json::json!({
966                        "consecutive_timeouts": state.consecutive_timeouts,
967                        "cooldown_remaining_ms":
968                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
969                        "probe_in_flight": state.probe_in_flight.is_some(),
970                    }),
971                ))
972            })
973            .collect::<serde_json::Map<String, serde_json::Value>>();
974        (!open.is_empty()).then_some(serde_json::Value::Object(open))
975    }
976}
977
978impl ControlHandler {
979    pub fn new(registry: Arc<Registry>) -> Self {
980        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
981    }
982
983    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
984        let counters = forwarding.counters();
985        // Taken from the forwarding table rather than created here, so that the
986        // breaker a `route.open` consults is the same one a module's
987        // registration resets, however many handlers are built over one table.
988        let route_bind_breakers = forwarding.route_bind_breakers();
989        let route_bind_concurrency = forwarding.route_bind_concurrency();
990        let route_outages = forwarding.route_outages();
991        Self {
992            registry,
993            activity: None,
994            activity_streams: Arc::new(Mutex::new(HashMap::new())),
995            forwarding,
996            process_liveness: None,
997            supervisor: SupervisorHandle::new(),
998            subc_capabilities: Arc::from([
999                CAP_MANIFEST_REGISTRATION.to_string(),
1000                CAP_CHANNEL_LIFECYCLE.to_string(),
1001                CAP_PING_PONG.to_string(),
1002                CAP_SESSION_ATTACH.to_string(),
1003                CAP_ADMISSION_FACTS_RELAY.to_string(),
1004                CAP_SCOPES_V1.to_string(),
1005                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1006                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1007                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1008                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1009                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1010                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1011                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1012                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1013                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1014                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1015                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1016            ]),
1017            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1018            route_bind_relay_timeouts: BTreeMap::new(),
1019            route_bind_breakers,
1020            route_bind_concurrency,
1021            route_outages,
1022            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1023            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1024            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1025            storage_config: None,
1026            machine_id: None,
1027            admission_facts_carrier_module_id: None,
1028            admission_facts_targets: None,
1029            scopes: Arc::new(RwLock::new(ScopeTable::new(
1030                crate::daemon_config::default_scope_authority_owners(),
1031            ))),
1032            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1033            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1034            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1035            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1036            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1037            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1038            local_connections: Arc::new(Mutex::new(HashSet::new())),
1039            operator_app_policy: Default::default(),
1040            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1041            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1042            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1043            #[cfg(test)]
1044            app_open_test_platform: None,
1045            #[cfg(test)]
1046            app_open_test_launch: None,
1047            #[cfg(windows)]
1048            windows_verify: None,
1049            #[cfg(windows)]
1050            windows_flash: None,
1051            #[cfg(windows)]
1052            windows_track: None,
1053            #[cfg(all(test, windows))]
1054            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1055            #[cfg(test)]
1056            caller_lookup: None,
1057            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1058            rescan: None,
1059            connected_clients: ConnectedClients::new(),
1060            local_endpoint_unavailable: None,
1061            counters,
1062            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1063            daemon_shutdown: DaemonShutdownFlag::default(),
1064            daemon_provenance: DaemonProvenanceFacts::default(),
1065            #[cfg(test)]
1066            control_dispatch_delay: None,
1067            #[cfg(test)]
1068            provenance_probe_override: None,
1069            #[cfg(test)]
1070            route_bind_relay_sent_hold: None,
1071            #[cfg(test)]
1072            hello_registry_to_forwarding_hold: None,
1073        }
1074    }
1075
1076    /// Set the central storage policy: registering modules then receive their
1077    /// resolved storage descriptor in HELLO_ACK.
1078    pub fn with_storage_config(
1079        mut self,
1080        storage_config: Option<crate::daemon_config::StorageConfig>,
1081    ) -> Self {
1082        self.storage_config = storage_config;
1083        self
1084    }
1085
1086    /// Open the ledger on its own writer thread; an unavailable sink does not
1087    /// prevent the daemon from serving routing or supervisor requests.
1088    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1089        self.activity = Some(crate::activity::ActivityLedger::open(
1090            run_dir.join("activity.jsonl"),
1091            incarnation,
1092        ));
1093        self.supervisor.set_activity(self.activity.clone());
1094        self
1095    }
1096
1097    /// Set the machine id served to every registering module (HELLO_ACK) and on
1098    /// `server.describe`.
1099    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1100        self.machine_id = machine_id;
1101        self
1102    }
1103
1104    /// Configure the exact reserved module and target ids permitted to relay
1105    /// opaque admission facts. Config-file loading validates this authority;
1106    /// this builder keeps the same policy available to embedded test daemons.
1107    pub fn with_admission_facts_config(
1108        mut self,
1109        carrier_module_id: Option<String>,
1110        targets: Option<Vec<String>>,
1111    ) -> Self {
1112        self.admission_facts_carrier_module_id = carrier_module_id;
1113        self.admission_facts_targets = targets;
1114        self
1115    }
1116
1117    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1118    /// Replaces the scope table with an empty one under the new list, so call it
1119    /// while building the handler, before any module can sync.
1120    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1121        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1122        self.scope_authority_owners = owners;
1123        self
1124    }
1125
1126    /// Set the startup policy before installing the rescan context or serving requests.
1127    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1128        self.privileged_operator_prompt = mode;
1129        self
1130    }
1131
1132    /// Override the route.bind relay timeout. Used by tests that assert the
1133    /// timeout path so they don't block on the production-safe default.
1134    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1135        self.route_bind_relay_timeout = timeout;
1136        self
1137    }
1138
1139    /// Install per-module route.bind relay budget overrides. A module id
1140    /// listed here wins over the daemon-wide default set via
1141    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1142    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1143    /// the same `Duration` the bind path will use.
1144    pub fn with_route_bind_relay_timeouts(
1145        mut self,
1146        timeouts: impl IntoIterator<Item = (String, Duration)>,
1147    ) -> Self {
1148        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1149        self
1150    }
1151
1152    /// Resolve the route.bind relay budget for a specific target module id.
1153    /// Per-module overrides win; the daemon-wide value (set via
1154    /// `with_route_bind_relay_timeout` or the built-in default) is the
1155    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1156    /// the same resolution `handle_route_open` will use.
1157    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1158        self.route_bind_relay_timeouts
1159            .get(module_id)
1160            .copied()
1161            .unwrap_or(self.route_bind_relay_timeout)
1162    }
1163
1164    /// Override the per-module bind-relay breaker policy.
1165    ///
1166    /// Used by tests, which cannot spend three production budgets opening a
1167    /// breaker or twenty seconds waiting for its cooldown. The production
1168    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1169    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1170    /// reasoning for the numbers.
1171    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1172        self.route_bind_breaker_threshold = threshold.max(1);
1173        self.route_bind_breaker_cooldown = cooldown;
1174        self
1175    }
1176
1177    #[cfg(test)]
1178    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1179        self.health_probe_timeout = timeout;
1180        self
1181    }
1182
1183    #[cfg(test)]
1184    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1185        self.control_dispatch_delay = Some(delay);
1186        self
1187    }
1188
1189    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1190    /// announce it is parked, once to be released. Build the barrier for two
1191    /// parties, the test being the other one.
1192    #[cfg(test)]
1193    pub(crate) fn with_route_bind_relay_sent_hold(
1194        mut self,
1195        hold: Arc<tokio::sync::Barrier>,
1196    ) -> Self {
1197        self.route_bind_relay_sent_hold = Some(hold);
1198        self
1199    }
1200
1201    /// A HELLO waits on `hold` twice between reserving its registry entry and
1202    /// installing its forwarding endpoint: once to announce it is parked, once
1203    /// to be released. Build the barrier for two parties, the test being the
1204    /// other one, and run the HELLO on a thread of its own.
1205    #[cfg(test)]
1206    pub(crate) fn with_hello_registry_to_forwarding_hold(
1207        mut self,
1208        hold: Arc<std::sync::Barrier>,
1209    ) -> Self {
1210        self.hello_registry_to_forwarding_hold = Some(hold);
1211        self
1212    }
1213
1214    #[cfg(test)]
1215    fn hold_between_registry_and_forwarding(&self) {
1216        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1217            hold.wait();
1218            hold.wait();
1219        }
1220    }
1221
1222    pub fn with_process_liveness(
1223        mut self,
1224        process_liveness: Arc<dyn ModuleProcessLiveness>,
1225    ) -> Self {
1226        self.process_liveness = Some(process_liveness);
1227        self
1228    }
1229
1230    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1231        self.supervisor = supervisor;
1232        self.supervisor.set_activity(self.activity.clone());
1233        self
1234    }
1235
1236    pub fn with_daemon_provenance(
1237        mut self,
1238        pid: u32,
1239        started_at_ms: u64,
1240        executable_path: Option<PathBuf>,
1241        build_git_sha: Option<String>,
1242        build_lock_digest: Option<String>,
1243    ) -> Self {
1244        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1245        let process_start_time = process_start_time(pid);
1246        self.daemon_provenance = DaemonProvenanceFacts {
1247            build: DaemonBuildProvenance {
1248                build_git_sha,
1249                build_lock_digest,
1250            },
1251            pid: Some(pid),
1252            started_at_ms: Some(started_at_ms),
1253            start_clock: None,
1254            executable_path,
1255            executable_identity,
1256            process_start_time,
1257            probe: ExecutableIdentityProbe::default(),
1258        };
1259        self
1260    }
1261
1262    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1263        self.daemon_provenance.start_clock = Some(clock);
1264        self
1265    }
1266
1267    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1268    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1269        self.scopes
1270            .write()
1271            .unwrap_or_else(|poisoned| poisoned.into_inner())
1272            .set_wall_clock(Arc::new(clock));
1273        self
1274    }
1275
1276    #[cfg(test)]
1277    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1278        self.provenance_probe_override = Some(result);
1279        self
1280    }
1281
1282    /// Install the configured module set and its reserved capability bindings.
1283    /// Bindings are configuration-scoped and may point at a provider that has not
1284    /// been installed yet, so this does not require the bound module to exist.
1285    pub fn with_capability_config(
1286        self,
1287        modules: impl IntoIterator<Item = (String, bool)>,
1288        reserved_capabilities: BTreeMap<String, String>,
1289    ) -> Self {
1290        self.capability_evaluator
1291            .configure(modules, reserved_capabilities);
1292        self
1293    }
1294
1295    /// Share the supervisor's daemon-shutdown flag with this handler, so
1296    /// requirement changes after the daemon has begun its announced shutdown
1297    /// are not reported as faults.
1298    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1299        self.daemon_shutdown = flag;
1300        self
1301    }
1302
1303    pub fn with_supervisor_rescan(
1304        mut self,
1305        supervisor: Supervisor,
1306        config_path: impl Into<PathBuf>,
1307        configured_port: Option<u16>,
1308    ) -> Self {
1309        self.rescan = Some(SupervisorRescanContext {
1310            supervisor,
1311            config_path: config_path.into(),
1312            configured_port,
1313            storage_config: self.storage_config.clone(),
1314            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1315            admission_facts_targets: self.admission_facts_targets.clone(),
1316            scope_authority_owners: self.scope_authority_owners.clone(),
1317            privileged_operator_prompt: self.privileged_operator_prompt,
1318            generation: Arc::default(),
1319        });
1320        self
1321    }
1322
1323    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1324        self.connected_clients = connected_clients;
1325        self
1326    }
1327
1328    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1329    /// an operator can see a bind failure without reading the daemon log.
1330    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1331        self.local_endpoint_unavailable = reason;
1332        self
1333    }
1334
1335    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1336        Arc::clone(&self.forwarding)
1337    }
1338
1339    pub(crate) fn counters(&self) -> DaemonCounters {
1340        self.counters.clone()
1341    }
1342
1343    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1344    /// requirement event without depending on an operator polling a status command.
1345    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1346        tokio::spawn(async move {
1347            loop {
1348                self.capability_evaluator
1349                    .wait_for_change_or_deadline()
1350                    .await;
1351                self.refresh_capability_requirements();
1352            }
1353        });
1354    }
1355
1356    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1357    /// deadline passed during sleep is enforced on the first tick after wake.
1358    /// While no live scope carries a deadline the loop waits for an accepted
1359    /// scope change instead of ticking, so a daemon without deadlines never
1360    /// wakes for expiry.
1361    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1362        tokio::spawn(async move {
1363            loop {
1364                let has_deadlines = self
1365                    .scopes
1366                    .read()
1367                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1368                    .has_deadlines();
1369                if !has_deadlines {
1370                    self.scope_deadline_added.notified().await;
1371                    continue;
1372                }
1373                tokio::time::sleep(Duration::from_secs(1)).await;
1374                if let Err(error) = self.sweep_expired_scopes() {
1375                    warn!(%error, "scope expiry sweep failed");
1376                }
1377            }
1378        });
1379    }
1380
1381    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1382        let mut table = self
1383            .scopes
1384            .write()
1385            .unwrap_or_else(|poisoned| poisoned.into_inner());
1386        let swept = table.sweep_expired();
1387        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1388        drop(table);
1389        self.close_scope_drained_routes(drained);
1390        Ok(())
1391    }
1392
1393    fn publish_scope_changes(
1394        &self,
1395        changes: &[crate::scopes::ScopeTagChange],
1396        expired: &[crate::scopes::ScopeExpired],
1397    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1398        if changes.is_empty() {
1399            return Ok(Vec::new());
1400        }
1401        let drained = self
1402            .forwarding
1403            .publish_scope_changes(changes)
1404            .map_err(RouterError::Forwarding)?;
1405        let mut counts = HashMap::new();
1406        for route in &drained {
1407            *counts
1408                .entry((
1409                    route.scope.owner.as_str(),
1410                    route.scope.scope_ref.as_str(),
1411                    route.scope.tag.scope_epoch,
1412                ))
1413                .or_insert(0usize) += 1;
1414        }
1415        for scope in expired {
1416            let routes_closed = counts
1417                .get(&(
1418                    scope.owner.as_str(),
1419                    scope.scope_ref.as_str(),
1420                    scope.scope_epoch,
1421                ))
1422                .copied()
1423                .unwrap_or(0);
1424            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1425                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1426                routes_closed, "scope expired");
1427        }
1428        Ok(drained)
1429    }
1430
1431    fn runtime_capability_snapshot(
1432        &self,
1433    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1434        let runtime = self
1435            .supervisor
1436            .list()
1437            .into_iter()
1438            .map(|module| {
1439                let status = module.status().map_err(|err| {
1440                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1441                })?;
1442                Ok(RuntimeModule {
1443                    module_id: status.module_id,
1444                    state: status.state,
1445                    enabled: status.enabled,
1446                })
1447            })
1448            .collect::<Result<Vec<_>, RouterError>>()?;
1449        // Include every registration, published or not. The capability
1450        // evaluator counts a module as providing its capabilities as soon as it
1451        // has registered, even before `catalog.list` shows it, so a HELLO in
1452        // progress does not change other modules' required-capability
1453        // verdicts. See `Registry::list_registered_modules`.
1454        let registrations = self.registry.list_registered_modules().map_err(|err| {
1455            RouterError::backend(
1456                0,
1457                0,
1458                format!("failed to list capability registrations: {err}"),
1459            )
1460        })?;
1461        let registrations = registrations
1462            .into_iter()
1463            .map(|registration| RegisteredModule {
1464                module_id: registration.manifest.module_id,
1465                module_version: registration.manifest.module_version,
1466                capabilities: registration.manifest.capabilities,
1467            })
1468            .collect();
1469        Ok((runtime, registrations))
1470    }
1471
1472    /// The capability side effects of a module becoming the active registration
1473    /// for its id: cache its manifest (logging how it differs from the cached
1474    /// one), run the deny census when its declarations call for one, and recompute the
1475    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1476    /// candidate's does not, and the supervisor does it at promotion instead,
1477    /// through [`crate::supervise::SwapPromotionObserver`].
1478    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1479        let cached_registration = RegisteredModule {
1480            module_id: registration.manifest.module_id.clone(),
1481            module_version: registration.manifest.module_version.clone(),
1482            capabilities: registration.manifest.capabilities.clone(),
1483        };
1484        let change = self.capability_evaluator.record_hello(&cached_registration);
1485        log_manifest_change(&cached_registration.module_id, &change);
1486        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1487            self.enforce_capability_denies();
1488        }
1489        self.refresh_capability_requirements();
1490    }
1491
1492    /// Point the shared supervisor handle at this handler for swap promotions.
1493    /// Called wherever a handler is put behind the `Arc` the router serves, so
1494    /// it can be held weakly.
1495    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1496        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1497            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1498        self.supervisor.set_swap_promotion_observer(observer);
1499    }
1500
1501    pub fn refresh_capability_requirements(&self) {
1502        match self.runtime_capability_snapshot() {
1503            Ok((runtime, registrations)) => {
1504                log_requirement_events(
1505                    self.capability_evaluator
1506                        .evaluate_now(&runtime, &registrations),
1507                    &self.daemon_shutdown,
1508                );
1509            }
1510            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1511        }
1512    }
1513
1514    /// Reconcile only live, attested route bindings after a capability deny edge
1515    /// or target claim was added. This is deliberately a control-plane census:
1516    /// the opaque forwarding hot path must not grow a per-frame capability check.
1517    fn enforce_capability_denies(&self) {
1518        let registrations = match self.registry.list_registered_modules() {
1519            Ok(snapshot) => snapshot,
1520            Err(err) => {
1521                warn!(error = %err, "failed to read registrations for capability deny census");
1522                return;
1523            }
1524        };
1525        let manifests = registrations
1526            .into_iter()
1527            .map(|registration| {
1528                (
1529                    registration.manifest.module_id.clone(),
1530                    registration.manifest,
1531                )
1532            })
1533            .collect::<BTreeMap<_, _>>();
1534        let census = match self.forwarding.route_census(None) {
1535            Ok(census) => census,
1536            Err(err) => {
1537                warn!(error = %err, "failed to read route census for capability deny enforcement");
1538                return;
1539            }
1540        };
1541
1542        for (target_module_id, routes) in census {
1543            let Some(target_manifest) = manifests.get(&target_module_id) else {
1544                continue;
1545            };
1546            let mut closed_routes = Vec::new();
1547            let mut goodbyes = Vec::new();
1548            for route in routes {
1549                let Principal::Reserved {
1550                    module_id: opening_module_id,
1551                } = &route.principal
1552                else {
1553                    continue;
1554                };
1555                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1556                    continue;
1557                };
1558                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1559                    continue;
1560                };
1561
1562                match self.forwarding.release_client_route(
1563                    route.goodbye_target.connection_id,
1564                    route.goodbye_target.channel,
1565                    route.goodbye_target.epoch,
1566                ) {
1567                    Ok(RouteRelease::Removed(module_goodbye)) => {
1568                        warn!(
1569                            opening_module_id,
1570                            target_module_id,
1571                            capability,
1572                            "force-closing route because an attested capability deny edge now matches"
1573                        );
1574                        goodbyes.push(module_goodbye);
1575                        goodbyes.push(route.goodbye_target.clone());
1576                        closed_routes.push(route);
1577                    }
1578                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1579                    Err(err) => warn!(
1580                        opening_module_id,
1581                        target_module_id,
1582                        capability,
1583                        error = %err,
1584                        "failed to force-close capability-denied route"
1585                    ),
1586                }
1587            }
1588
1589            if closed_routes.is_empty() {
1590                continue;
1591            }
1592            send_route_control_pushes(
1593                &self.forwarding,
1594                closed_routes,
1595                ClientControlPush::RouteClosed {
1596                    module_id: target_module_id,
1597                    channels: Vec::new(),
1598                    reason: RouteCloseReason::CapabilityDenied,
1599                    drained: false,
1600                    abandoned: 0,
1601                    excluded_subscriptions: 0,
1602                    terminal: Some(false),
1603                },
1604            );
1605            // route.closed records the reason; the route GOODBYE ends the
1606            // client's handle and settles any calls still pending on it.
1607            self.emit_route_goodbyes(goodbyes);
1608        }
1609    }
1610
1611    /// Why a registered module is not accepting new route binds, or `None` when
1612    /// it is. This is the module's effective readiness: its declared readiness
1613    /// first, then every `need: required` capability it declares evaluating to
1614    /// `provided`. `route.open` and `catalog.list` both read it here so the
1615    /// catalog never reports a module routable that `route.open` would refuse.
1616    fn not_ready_reason(
1617        &self,
1618        registration: &crate::registry::ModuleRegistration,
1619    ) -> Option<NotReadyReason> {
1620        if !registration.ready {
1621            return Some(NotReadyReason {
1622                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1623                capability: None,
1624            });
1625        }
1626        self.first_unprovided_required_capability(registration)
1627            .map(|capability| NotReadyReason {
1628                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1629                capability: Some(capability),
1630            })
1631    }
1632
1633    /// The lexicographically first capability this registration declares
1634    /// `need: required` whose evaluator verdict is not `provided`.
1635    ///
1636    /// The verdicts are the capability evaluator's own; nothing here decides
1637    /// what "provided" means. The evaluator counts a capability provided as
1638    /// soon as a module claiming it has REGISTERED, not once that module is
1639    /// ready. That distinction is what keeps two modules that require each
1640    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1641    /// is ready", each would wait for the other to become ready first and
1642    /// neither ever would. Do not tighten it to readiness.
1643    ///
1644    /// A required capability with no verdict at all means this registration's
1645    /// HELLO or catalog.update landed after the last recompute; recompute once
1646    /// rather than let a missing verdict read as either answer. If it is still
1647    /// missing (the recompute itself failed) the capability counts as
1648    /// unprovided: the refusal is retryable, and routing a module whose
1649    /// required provider is unknown is the outcome this check exists to stop.
1650    fn first_unprovided_required_capability(
1651        &self,
1652        registration: &crate::registry::ModuleRegistration,
1653    ) -> Option<String> {
1654        let required = registration
1655            .manifest
1656            .capabilities
1657            .iter()
1658            .flat_map(|declarations| declarations.requires.iter())
1659            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1660            .map(|requirement| requirement.capability.as_str())
1661            .collect::<BTreeSet<_>>();
1662        if required.is_empty() {
1663            return None;
1664        }
1665        let module_id = registration.manifest.module_id.as_str();
1666        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1667        if required
1668            .iter()
1669            .any(|capability| verdict(capability).is_none())
1670        {
1671            self.refresh_capability_requirements();
1672        }
1673        required
1674            .into_iter()
1675            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1676            .map(str::to_string)
1677    }
1678
1679    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1680        self.capability_evaluator
1681            .statuses()
1682            .into_iter()
1683            .map(capability_requirement_status)
1684            .collect()
1685    }
1686
1687    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1688    /// registration-release watch. The signal is what the supervisor waits on
1689    /// before spawning a replacement, so it must only fire once forwarding
1690    /// teardown is also done (see [`Self::cleanup_connection`] /
1691    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1692    /// admitted but its forwarding endpoint could not be installed.
1693    fn deregister_connection(
1694        &self,
1695        connection_id: ConnectionId,
1696        reason: RegistrationEndReason,
1697    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1698        self.registry
1699            .deregister_connection_with_reason(connection_id, reason)
1700    }
1701
1702    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1703        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1704            return None;
1705        }
1706        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1707            parse_client_control_request(&frame.body)
1708        else {
1709            return None;
1710        };
1711        Some(target_module_id(&target).to_string())
1712    }
1713
1714    pub(crate) fn route_open_capacity_refusal(
1715        &self,
1716        ctx: &RouteCtx,
1717        frame: &Frame,
1718        target_module_id: &str,
1719        in_flight: usize,
1720        limit: usize,
1721    ) -> Result<Frame, RouterError> {
1722        // Sent before the request reaches `handle_route_open`, so the caller
1723        // fields come from the raw frame here.
1724        let caller = self.route_open_caller_from_frame(frame);
1725        self.route_open_admission_refusal_frame(
1726            ctx,
1727            &caller,
1728            frame,
1729            target_module_id,
1730            "open_admission_full",
1731            (in_flight, limit),
1732            format!(
1733                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1734            ),
1735        )
1736    }
1737
1738    fn route_open_target_capacity_refusal(
1739        &self,
1740        ctx: &RouteCtx,
1741        caller: &RouteOpenCaller,
1742        frame: &Frame,
1743        target_module_id: &str,
1744        in_flight: usize,
1745    ) -> Result<Frame, RouterError> {
1746        self.route_open_admission_refusal_frame(
1747            ctx,
1748            caller,
1749            frame,
1750            target_module_id,
1751            "target_binds_full",
1752            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1753            format!(
1754                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1755            ),
1756        )
1757    }
1758
1759    /// Admission pressure clears as existing binds settle, so its refusal must
1760    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1761    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1762    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1763    /// cannot currently reach its target; `module_timeout` would falsely claim
1764    /// that a wait expired. A new, cleaner code would be terminal to deployed
1765    /// clients, so it requires a client-tolerance rollout before daemon emission.
1766    #[allow(clippy::too_many_arguments)]
1767    fn route_open_admission_refusal_frame(
1768        &self,
1769        ctx: &RouteCtx,
1770        caller: &RouteOpenCaller,
1771        frame: &Frame,
1772        target_module_id: &str,
1773        reason: &'static str,
1774        (in_flight, limit): (usize, usize),
1775        message: impl Into<String>,
1776    ) -> Result<Frame, RouterError> {
1777        let code = error_codes::TARGET_UNAVAILABLE;
1778        self.counters.increment_route_open_refused(code);
1779        info!(
1780            target: "control",
1781            code,
1782            reason,
1783            module_id = ?target_module_id,
1784            connection_id = ctx.connection_id.get(),
1785            principal = %caller.principal,
1786            scope = %caller.scope,
1787            harness = %caller.harness,
1788            in_flight,
1789            limit,
1790            "route.open refused"
1791        );
1792        control_error_frame(frame, code, message.into())
1793    }
1794
1795    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1796    ///
1797    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1798    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1799    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1800    #[cfg(test)]
1801    pub fn handle_control(
1802        &self,
1803        connection_id: ConnectionId,
1804        frame: Frame,
1805    ) -> Result<Vec<Frame>, RouterError> {
1806        match frame.header.ty {
1807            FrameType::Ping => Ok(vec![pong(&frame)?]),
1808            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1809            FrameType::Goodbye => self.handle_goodbye(connection_id),
1810            ty => Ok(vec![control_error_frame(
1811                &frame,
1812                "unsupported_control_frame",
1813                format!("unsupported channel-0 frame {ty:?}"),
1814            )?]),
1815        }
1816    }
1817
1818    pub async fn handle_control_frame(
1819        &self,
1820        ctx: &RouteCtx,
1821        frame: Frame,
1822    ) -> Result<Vec<Frame>, RouterError> {
1823        self.handle_control_frame_timed(ctx, frame, None).await
1824    }
1825
1826    #[cfg(any(test, feature = "test-support"))]
1827    pub fn with_test_operator_app_policy(
1828        mut self,
1829        policy: crate::operator_apps::TestOperatorAppPolicy,
1830    ) -> Self {
1831        self.operator_app_policy = policy.into();
1832        self
1833    }
1834
1835    /// Inject the Windows signature check for tests. The injected check
1836    /// answers the verified version string or the rejection reason; the
1837    /// daemon enforces the floor itself on the returned version.
1838    #[cfg(all(windows, any(test, feature = "test-support")))]
1839    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1840        self.windows_verify = Some(verify);
1841        self
1842    }
1843
1844    /// Record which pids the daemon would flash for `already_running`.
1845    #[cfg(all(windows, any(test, feature = "test-support")))]
1846    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1847        self.windows_flash = Some(flash);
1848        self
1849    }
1850
1851    /// Inject a tracking failure for tests: makes `track_launched_process`
1852    /// fail so a test can prove a launch that cannot be tracked is stopped,
1853    /// never left running.
1854    #[cfg(all(windows, any(test, feature = "test-support")))]
1855    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1856        self.windows_track = Some(track);
1857        self
1858    }
1859
1860    pub(crate) fn record_captured_peer(
1861        &self,
1862        connection: ConnectionId,
1863        peer: subc_os::local_peer::CapturedPeer,
1864    ) {
1865        self.record_observed_peer(connection, peer.observation().clone());
1866        #[cfg(windows)]
1867        if let Some(stamp) = self.stamp_windows_launch(&peer) {
1868            self.operator_apps
1869                .lock()
1870                .unwrap_or_else(|p| p.into_inner())
1871                .insert(connection, stamp);
1872        }
1873        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1874            self.operator_apps
1875                .lock()
1876                .unwrap_or_else(|p| p.into_inner())
1877                .insert(connection, stamp);
1878        }
1879    }
1880
1881    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1882        self.local_connections
1883            .lock()
1884            .unwrap_or_else(|p| p.into_inner())
1885            .insert(connection);
1886    }
1887
1888    // Keep the connection-level delivery decision shared with route.bind. The
1889    // target's capability opt-in is separate from recognition of the caller.
1890    fn connection_operator_app(
1891        &self,
1892        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1893    ) -> Option<subc_protocol::peer::OperatorApp> {
1894        stamp
1895            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1896            .map(|stamp| stamp.app.clone())
1897    }
1898
1899    fn handle_connection_describe(
1900        &self,
1901        ctx: &RouteCtx,
1902        frame: Frame,
1903    ) -> Result<Vec<Frame>, RouterError> {
1904        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1905            return Ok(vec![control_error_frame(
1906                &frame,
1907                "invalid_control_body",
1908                "malformed connection.describe body",
1909            )?]);
1910        }
1911        let local = self
1912            .local_connections
1913            .lock()
1914            .unwrap_or_else(|p| p.into_inner())
1915            .contains(&ctx.connection_id);
1916        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1917        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1918        Ok(vec![control_response_body_frame(
1919            &frame,
1920            &subc_control::ConnectionDescribeReply::new(local, operator_app),
1921            ops::CONNECTION_DESCRIBE,
1922        )?])
1923    }
1924
1925    pub(crate) fn record_observed_peer(
1926        &self,
1927        connection: ConnectionId,
1928        peer: subc_os::local_peer::ObservedPeer,
1929    ) {
1930        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1931        let peer = match peer {
1932            OsPeer::Unobserved => ObservedPeer::unobserved(),
1933            OsPeer::Observed(peer) => {
1934                let observed = match peer.user {
1935                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1936                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1937                };
1938                observed
1939                    .with_exe(
1940                        peer.exe
1941                            .and_then(|path| path.into_os_string().into_string().ok()),
1942                    )
1943                    .with_signing_id(peer.signing_id)
1944            }
1945        };
1946        self.observed_peers
1947            .lock()
1948            .unwrap_or_else(|p| p.into_inner())
1949            .insert(connection, peer);
1950    }
1951
1952    pub(crate) fn record_caller_endpoints(
1953        &self,
1954        connection: ConnectionId,
1955        endpoints: Option<(SocketAddr, SocketAddr)>,
1956    ) {
1957        let mut callers = self
1958            .caller_endpoints
1959            .lock()
1960            .unwrap_or_else(|p| p.into_inner());
1961        callers.remove(&connection);
1962        if let Some(endpoints) = endpoints {
1963            callers.insert(connection, endpoints);
1964        }
1965    }
1966
1967    #[cfg(test)]
1968    pub(crate) fn with_caller_lookup_for_test(
1969        mut self,
1970        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1971    ) -> Self {
1972        self.caller_lookup = Some(Arc::new(lookup));
1973        self
1974    }
1975
1976    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1977        let endpoints = self
1978            .caller_endpoints
1979            .lock()
1980            .unwrap_or_else(|p| p.into_inner())
1981            .get(&connection)
1982            .copied();
1983        #[cfg(test)]
1984        let lookup = self.caller_lookup.clone();
1985        tokio::task::spawn_blocking(move || {
1986            #[cfg(test)]
1987            if let Some(lookup) = lookup {
1988                return lookup(endpoints);
1989            }
1990            endpoints
1991                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1992                .unwrap_or_default()
1993        })
1994        .await
1995        .unwrap_or_default()
1996    }
1997
1998    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1999        frame.header.channel == 0
2000            && frame.header.ty == FrameType::Request
2001            && self
2002                .forwarding
2003                .module_endpoint_for_connection(ctx.connection_id)
2004                .is_ok_and(|endpoint| endpoint.is_none())
2005            && parse_client_control_request(&frame.body)
2006                .ok()
2007                .is_some_and(|request| privileged_operation(&request).is_some())
2008    }
2009
2010    pub(crate) async fn handle_control_frame_timed(
2011        &self,
2012        ctx: &RouteCtx,
2013        frame: Frame,
2014        dispatch_started_at: Option<StdInstant>,
2015    ) -> Result<Vec<Frame>, RouterError> {
2016        match frame.header.ty {
2017            FrameType::Ping => Ok(vec![pong(&frame)?]),
2018            FrameType::Hello => {
2019                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2020            }
2021            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2022            FrameType::Cancel => {
2023                // A Cancel on channel 0 names a waiting operator.confirm or a
2024                // subscription. Successful cancellation has no ordinary reply.
2025                if self
2026                    .forwarding
2027                    .operator_confirms()
2028                    .cancel(ctx.connection_id, frame.header.corr)
2029                    || self
2030                        .supervisor
2031                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2032                    || self.cancel_activity_subscription(ctx, &frame)?
2033                {
2034                    Ok(Vec::new())
2035                } else {
2036                    Ok(vec![control_error_frame(
2037                        &frame,
2038                        "unknown_subscription",
2039                        "no subscription has this correlation id",
2040                    )?])
2041                }
2042            }
2043            FrameType::Request => {
2044                // This additive operation is not part of the existing exhaustive
2045                // module-control enum. Probe the op before decoding that enum.
2046                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2047                if op
2048                    .as_ref()
2049                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2050                {
2051                    return self.handle_activity_record(ctx, frame).await;
2052                }
2053                if op.as_ref().is_some_and(|probe| {
2054                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2055                }) {
2056                    return self.handle_activity_read(ctx, frame).await;
2057                }
2058                if op
2059                    .as_ref()
2060                    .is_some_and(|probe| probe.op == "operator.confirm")
2061                {
2062                    return self.handle_operator_confirm(ctx, frame);
2063                }
2064                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2065                    return self
2066                        .observe_control_wait(
2067                            ctx,
2068                            None,
2069                            ops::APP_OPEN,
2070                            self.handle_app_open(ctx, frame),
2071                        )
2072                        .await;
2073                }
2074                if op
2075                    .as_ref()
2076                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2077                {
2078                    return self.handle_connection_describe(ctx, frame);
2079                }
2080                if op
2081                    .as_ref()
2082                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2083                {
2084                    return self.handle_scope_served_by(ctx.connection_id, frame);
2085                }
2086                if self
2087                    .forwarding
2088                    .module_endpoint_for_connection(ctx.connection_id)
2089                    .map_err(RouterError::Forwarding)?
2090                    .is_some()
2091                {
2092                    if !is_known_module_request_op(&frame.body) {
2093                        return Ok(vec![control_error_frame(
2094                            &frame,
2095                            "unsupported_control_frame",
2096                            "module-originated channel-0 REQUEST is not supported",
2097                        )?]);
2098                    }
2099                    let request = match parse_module_control_request_from_module(&frame.body) {
2100                        Ok(request) => request,
2101                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2102                            return Ok(vec![control_error_frame(
2103                                &frame,
2104                                "unsupported_control_frame",
2105                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2106                            )?])
2107                        }
2108                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2109                            return Ok(vec![control_error_frame(
2110                                &frame,
2111                                "invalid_control_body",
2112                                format!("malformed module control body: {err}"),
2113                            )?])
2114                        }
2115                    };
2116                    let op = module_control_request_op(&request);
2117                    let corr = frame.header.corr;
2118                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2119                    let result =
2120                        self.handle_module_control_request(ctx.connection_id, frame, request);
2121                    log_slow_control_dispatch(
2122                        dispatch_started_at,
2123                        op,
2124                        ctx.connection_id,
2125                        corr,
2126                        None,
2127                    );
2128                    return result;
2129                }
2130
2131                if is_known_module_request_op(&frame.body) {
2132                    return Ok(vec![control_error_frame(
2133                        &frame,
2134                        "not_registered",
2135                        "catalog.update requires an active module registration owned by this connection",
2136                    )?]);
2137                }
2138
2139                let request = match parse_client_control_request(&frame.body) {
2140                    Ok(request) => request,
2141                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2142                        return Ok(vec![control_error_frame(
2143                            &frame,
2144                            "unknown_control_op",
2145                            format!("unknown client control op: {err}"),
2146                        )?])
2147                    }
2148                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2149                        return Ok(vec![control_error_frame(
2150                            &frame,
2151                            "invalid_control_body",
2152                            format!("malformed client control body: {err}"),
2153                        )?])
2154                    }
2155                };
2156                let op = client_control_request_op(&request);
2157                let corr = frame.header.corr;
2158                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2159                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2160                #[cfg(test)]
2161                if let Some(delay) = self.control_dispatch_delay {
2162                    tokio::time::sleep(delay).await;
2163                }
2164                let result = self
2165                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2166                    .await;
2167                if let Some(timings) = route_open_timings.as_mut() {
2168                    timings.finish();
2169                }
2170                log_slow_control_dispatch(
2171                    dispatch_started_at,
2172                    op,
2173                    ctx.connection_id,
2174                    corr,
2175                    route_open_timings.as_ref(),
2176                );
2177                result
2178            }
2179            FrameType::Push => {
2180                let Some(endpoint) = self
2181                    .forwarding
2182                    .module_endpoint_for_connection(ctx.connection_id)
2183                    .map_err(RouterError::Forwarding)?
2184                else {
2185                    return Ok(vec![control_error_frame(
2186                        &frame,
2187                        "unsupported_control_frame",
2188                        "client-originated channel-0 PUSH is not supported",
2189                    )?]);
2190                };
2191                self.handle_status_update(endpoint, frame)
2192            }
2193            FrameType::Response | FrameType::Error
2194                if self
2195                    .forwarding
2196                    .module_endpoint_for_connection(ctx.connection_id)
2197                    .map_err(RouterError::Forwarding)?
2198                    .is_some() =>
2199            {
2200                self.handle_module_relay_response(ctx.connection_id, frame)
2201            }
2202            ty => Ok(vec![control_error_frame(
2203                &frame,
2204                "unsupported_control_frame",
2205                format!("unsupported channel-0 frame {ty:?}"),
2206            )?]),
2207        }
2208    }
2209
2210    pub fn cleanup_connection(
2211        &self,
2212        connection_id: ConnectionId,
2213    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2214        self.local_connections
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner())
2217            .remove(&connection_id);
2218        self.caller_endpoints
2219            .lock()
2220            .unwrap_or_else(|p| p.into_inner())
2221            .remove(&connection_id);
2222        self.build_class_warned
2223            .lock()
2224            .unwrap_or_else(|p| p.into_inner())
2225            .remove(&connection_id);
2226        self.observed_peers
2227            .lock()
2228            .unwrap_or_else(|p| p.into_inner())
2229            .remove(&connection_id);
2230        self.operator_apps
2231            .lock()
2232            .unwrap_or_else(|p| p.into_inner())
2233            .remove(&connection_id);
2234        self.forwarding
2235            .operator_confirms()
2236            .connection_closed(connection_id);
2237        self.cleanup_connection_with_end_reason(
2238            connection_id,
2239            RegistrationEndReason::ConnectionClosed,
2240        )
2241    }
2242
2243    fn cleanup_connection_with_end_reason(
2244        &self,
2245        connection_id: ConnectionId,
2246        requested_reason: RegistrationEndReason,
2247    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2248        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2249            self.registry
2250                .get_module_by_connection(connection_id)?
2251                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2252                .and_then(|module| module.registration_end_reason().ok().flatten())
2253                .unwrap_or(requested_reason)
2254        } else {
2255            requested_reason
2256        };
2257        let crash_closed = self
2258            .registry
2259            .get_module_by_connection(connection_id)?
2260            .and_then(|registration| {
2261                self.forwarding
2262                    .module_endpoint_for_connection(connection_id)
2263                    .ok()
2264                    .flatten()
2265                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2266                    .map(|routes| (registration.manifest.module_id, routes))
2267            });
2268        let crash_closed = crash_closed.map(|(module_id, routes)| {
2269            let terminal = match self.supervisor.get(&module_id) {
2270                None => false,
2271                Some(module) => match module.will_recover_after_connection_loss() {
2272                    Ok(will_recover) => !will_recover,
2273                    Err(err) => {
2274                        warn!(
2275                            %module_id,
2276                            error = %err,
2277                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2278                        );
2279                        false
2280                    }
2281                },
2282            };
2283            // The forwarding table gates all providers at the start of daemon
2284            // shutdown, before their connections are closed. An ordinary
2285            // module disconnect still reports crash if that gate is not set.
2286            let (reason, drained) = match self
2287                .forwarding
2288                .daemon_drain_result_for_connection(connection_id)
2289            {
2290                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2291                Ok(None) => (RouteCloseReason::Crash, false),
2292                Err(err) => {
2293                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2294                    (RouteCloseReason::Crash, false)
2295                }
2296            };
2297            (module_id, routes, reason, drained, terminal)
2298        });
2299        let registrations = self.deregister_connection(connection_id, end_reason);
2300        let cleanup = if crash_closed.is_some() {
2301            self.forwarding.cleanup_connection_counted(connection_id)
2302        } else {
2303            // A module connection's teardown needs the count of abandoned
2304            // route.bind relays for its route.closed notice below. Any other
2305            // connection, such as a client's, sends no such notice and needs
2306            // only its routes released, so it uses the route-only wrapper and
2307            // reports zero.
2308            self.forwarding
2309                .cleanup_connection(connection_id)
2310                .map(|released| crate::forwarding::ConnectionCleanup {
2311                    released,
2312                    abandoned_relays: 0,
2313                })
2314        };
2315        // The route.closed push waits for forwarding teardown because only
2316        // teardown knows how many pending route.bind relays it aborted. It still
2317        // goes out before the GOODBYEs for the released routes, and its targets
2318        // were captured above, before teardown removed those routes.
2319        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2320            let abandoned = cleanup
2321                .as_ref()
2322                .map_or(0, |cleanup| cleanup.abandoned_relays);
2323            send_route_control_pushes(
2324                &self.forwarding,
2325                routes,
2326                ClientControlPush::RouteClosed {
2327                    module_id,
2328                    channels: Vec::new(),
2329                    reason,
2330                    drained,
2331                    abandoned,
2332                    excluded_subscriptions: 0,
2333                    terminal: Some(terminal),
2334                },
2335            );
2336        }
2337        if let Ok(cleanup) = cleanup {
2338            self.emit_route_goodbyes(cleanup.released);
2339        }
2340        // Signal the registration-release watch only now that BOTH registry and
2341        // forwarding teardown are done, so a supervisor waiting to spawn a
2342        // replacement never observes release while old routes still exist.
2343        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2344            crate::supervise::notify_registration_release();
2345            self.capability_evaluator.wake_deadline_loop();
2346            self.refresh_capability_requirements();
2347        }
2348        self.supervisor.remove_spawn_subscribers(connection_id);
2349        self.activity_streams
2350            .lock()
2351            .unwrap_or_else(|p| p.into_inner())
2352            .retain(|(id, _), task| {
2353                if *id == connection_id {
2354                    task.abort();
2355                    false
2356                } else {
2357                    true
2358                }
2359            });
2360        // Sync authority dies with its connection, so the owner's next
2361        // connection can take it; the owner's scopes stay as they are.
2362        self.hello_launch_nonces
2363            .lock()
2364            .unwrap_or_else(|poisoned| poisoned.into_inner())
2365            .forget(connection_id);
2366        self.scopes
2367            .write()
2368            .unwrap_or_else(|poisoned| poisoned.into_inner())
2369            .release_connection(connection_id);
2370        registrations
2371    }
2372
2373    pub(crate) fn handle_route_goodbye(
2374        &self,
2375        connection_id: ConnectionId,
2376        route_channel: u16,
2377        route_epoch: u32,
2378    ) -> Result<bool, RouterError> {
2379        debug!(
2380            connection_id = connection_id.get(),
2381            route_channel, route_epoch, "handling route GOODBYE"
2382        );
2383        let RouteRelease::Removed(released_route) = self
2384            .forwarding
2385            .release_client_route(connection_id, route_channel, route_epoch)
2386            .map_err(RouterError::Forwarding)?
2387        else {
2388            return Ok(false);
2389        };
2390        self.emit_route_goodbyes(vec![released_route]);
2391        Ok(true)
2392    }
2393
2394    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2395        for released in released_routes {
2396            let frame = match Frame::build_with_version(
2397                released.negotiated_ver,
2398                FrameType::Goodbye,
2399                control_flags(),
2400                released.channel,
2401                released.epoch,
2402                0,
2403                Vec::new(),
2404            ) {
2405                Ok(frame) => frame,
2406                Err(err) => {
2407                    warn!(
2408                        route_channel = released.channel,
2409                        error = %err,
2410                        "failed to build route GOODBYE frame"
2411                    );
2412                    continue;
2413                }
2414            };
2415            if !released.close_on_delivery_failure() {
2416                crate::forwarding::send_module_route_goodbye(
2417                    &self.counters,
2418                    &released.sink,
2419                    frame,
2420                    released.module_id.as_deref(),
2421                    "client route released",
2422                );
2423                continue;
2424            }
2425            if let Err(err) = released.sink.try_send(frame) {
2426                warn!(
2427                    target_connection_id = released.connection_id.get(),
2428                    route_channel = released.channel,
2429                    error = %err,
2430                    "route GOODBYE was not delivered to client; closing target connection"
2431                );
2432                if self
2433                    .forwarding
2434                    .escalate_client_delivery_failure(
2435                        released.connection_id,
2436                        released.channel,
2437                        released.epoch,
2438                        CloseReason::new(
2439                            "route_goodbye_delivery_failed",
2440                            format!(
2441                                "failed to enqueue route GOODBYE for channel {}: {err}",
2442                                released.channel
2443                            ),
2444                        ),
2445                        crate::forwarding::UndeliveredFrame {
2446                            module_id: released.module_id.as_deref(),
2447                            sink: &released.sink,
2448                        },
2449                    )
2450                    .unwrap_or(false)
2451                {
2452                    self.counters.increment_goodbye_relay_client_failed();
2453                }
2454            }
2455        }
2456    }
2457
2458    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2459    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2460    /// own commit failed after the module had already accepted). Without this, a
2461    /// module that accepts late keeps a binding subc has torn down, so a later
2462    /// frame on that module channel could misdeliver if the channel is reused.
2463    ///
2464    /// Never closes the shared module connection on failure: a dropped notification
2465    /// only wastes a bounded amount of warm module-side state, which the module's
2466    /// own idle reaper reclaims. Only call this once the route.bind relay was
2467    /// actually enqueued to the module — if the relay send itself failed, the
2468    /// module never created a binding and there is nothing to tear down.
2469    fn send_abandoned_route_bind_goodbye(
2470        &self,
2471        module_sink: &crate::FrameSink,
2472        negotiated_ver: u8,
2473        module_channel: u16,
2474        module_epoch: u32,
2475    ) {
2476        let frame = match Frame::build_with_version(
2477            negotiated_ver,
2478            FrameType::Goodbye,
2479            control_flags(),
2480            module_channel,
2481            module_epoch,
2482            0,
2483            Vec::new(),
2484        ) {
2485            Ok(frame) => frame,
2486            Err(err) => {
2487                warn!(
2488                    route_channel = module_channel,
2489                    error = %err,
2490                    "failed to build GOODBYE for abandoned route.bind"
2491                );
2492                return;
2493            }
2494        };
2495        crate::forwarding::send_module_route_goodbye(
2496            &self.counters,
2497            module_sink,
2498            frame,
2499            None,
2500            "abandoned route.bind",
2501        );
2502    }
2503
2504    fn handle_hello(
2505        &self,
2506        connection_id: ConnectionId,
2507        sink: Option<crate::FrameSink>,
2508        frame: Frame,
2509    ) -> Result<Vec<Frame>, RouterError> {
2510        debug!(
2511            connection_id = connection_id.get(),
2512            corr = frame.header.corr,
2513            "handling HELLO"
2514        );
2515        // A module connection has one identity for its entire lifetime. A second
2516        // registration would leave the old registry owner behind while replacing
2517        // its forwarding endpoint and launch nonce.
2518        if self
2519            .registry
2520            .get_module_by_connection(connection_id)
2521            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2522            .is_some()
2523        {
2524            return Ok(vec![control_error_frame(
2525                &frame,
2526                "invalid_hello",
2527                "connection is already registered as a module",
2528            )?]);
2529        }
2530        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2531            Ok(value) => value,
2532            Err(err) => {
2533                return Ok(vec![control_error_frame(
2534                    &frame,
2535                    "invalid_hello",
2536                    format!("malformed HELLO body: {err}"),
2537                )?])
2538            }
2539        };
2540        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2541            return Ok(vec![control_error_frame(
2542                &frame,
2543                "invalid_capability_grammar",
2544                err.to_string(),
2545            )?]);
2546        }
2547        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2548            return Ok(vec![control_error_frame(
2549                &frame,
2550                "invalid_manifest",
2551                err.to_string(),
2552            )?]);
2553        }
2554        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2555            return Ok(vec![control_error_frame(
2556                &frame,
2557                "invalid_event_declaration",
2558                err.to_string(),
2559            )?]);
2560        }
2561        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2562            return Ok(vec![control_error_frame(
2563                &frame,
2564                "invalid_manifest",
2565                err.to_string(),
2566            )?]);
2567        }
2568        if let Some(provenance) = hello_value
2569            .get("manifest")
2570            .and_then(|manifest| manifest.get("provenance"))
2571        {
2572            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2573                return Ok(vec![control_error_frame(
2574                    &frame,
2575                    "invalid_manifest",
2576                    format!("malformed manifest provenance: {err}"),
2577                )?]);
2578            }
2579        }
2580        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2581            let module_id = hello_value["manifest"]["module_id"]
2582                .as_str()
2583                .unwrap_or("<unknown>");
2584            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2585        }
2586        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2587            let module_id = hello_value["manifest"]["module_id"]
2588                .as_str()
2589                .unwrap_or("<unknown>");
2590            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2591        }
2592        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2593            let module_id = hello_value["manifest"]["module_id"]
2594                .as_str()
2595                .unwrap_or("<unknown>");
2596            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2597        }
2598        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2599            Ok(hello) => hello,
2600            Err(err) => {
2601                return Ok(vec![control_error_frame(
2602                    &frame,
2603                    "invalid_hello",
2604                    format!("malformed HELLO body: {err}"),
2605                )?])
2606            }
2607        };
2608
2609        if hello.protocol_ver != hello.manifest.protocol_ver {
2610            return Ok(vec![control_error_frame(
2611                &frame,
2612                "invalid_manifest",
2613                format!(
2614                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2615                    hello.protocol_ver, hello.manifest.protocol_ver
2616                ),
2617            )?]);
2618        }
2619
2620        if hello.manifest.module_id.trim().is_empty() {
2621            return Ok(vec![control_error_frame(
2622                &frame,
2623                "invalid_manifest",
2624                "manifest module_id must not be empty",
2625            )?]);
2626        }
2627
2628        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2629            Ok(negotiated_ver) => negotiated_ver,
2630            Err(message) => {
2631                return Ok(vec![control_error_frame(
2632                    &frame,
2633                    "version_unsupported",
2634                    message,
2635                )?])
2636            }
2637        };
2638
2639        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2640        // swap is open for this id, the only HELLO admitted as a second process
2641        // is the one carrying the candidate's launch nonce (the swap token), and
2642        // it registers into the candidate slot rather than being refused as a
2643        // duplicate. Run after the reserved gate, a reserved module's candidate
2644        // would be refused `reserved_module` for presenting a nonce that gate
2645        // does not know. See `SupervisorHandle::swap_hello_admission`.
2646        let swap_admission = self
2647            .supervisor
2648            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2649        if swap_admission == SwapHelloAdmission::Refused {
2650            warn!(
2651                module_id = %hello.manifest.module_id,
2652                connection_id = connection_id.get(),
2653                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2654            );
2655            return Ok(vec![control_error_frame(
2656                &frame,
2657                "swap_token_invalid",
2658                format!(
2659                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2660                    hello.manifest.module_id
2661                ),
2662            )?]);
2663        }
2664        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2665
2666        // A swap candidate proves its identity with the candidate nonce above.
2667        // Other registrations receive this catalog fact only when reserved-module
2668        // policy required and accepted a nonce; unreserved registrations require
2669        // no nonce check.
2670        let nonce_verified = if swap_candidate {
2671            true
2672        } else {
2673            match self.supervisor.reserved_hello_verification(
2674                &hello.manifest.module_id,
2675                hello.launch_nonce.as_deref(),
2676            ) {
2677                Ok(verified) => verified,
2678                Err(rejection) => {
2679                    let message = match rejection {
2680                        ReservedHelloRejection::Exact { module_id } => format!(
2681                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2682                        ),
2683                        ReservedHelloRejection::Prefix {
2684                            prefix,
2685                            owner_module_id,
2686                        } => format!(
2687                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2688                            hello.manifest.module_id
2689                        ),
2690                    };
2691                    return Ok(vec![control_error_frame(
2692                        &frame,
2693                        "reserved_module",
2694                        message,
2695                    )?]);
2696                }
2697            }
2698        };
2699
2700        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2701            &hello.manifest.module_id,
2702            hello.manifest.capabilities.as_ref(),
2703        );
2704        if let Some(refusal) = reserved_capability_refusals.first() {
2705            let capability = refusal.capability.clone();
2706            let bound_module = refusal.claimants[0].clone();
2707            log_duplicate_claim_events(reserved_capability_refusals);
2708            return Ok(vec![control_error_frame(
2709                &frame,
2710                "reserved_capability",
2711                format!(
2712                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2713                    capability, bound_module, hello.manifest.module_id
2714                ),
2715            )?]);
2716        }
2717
2718        // A connection that already opened client routes must not also register as
2719        // a module: cleanup would then release only one side and leak the other.
2720        if self
2721            .forwarding
2722            .connection_has_client_routes(connection_id)
2723            .map_err(RouterError::Forwarding)?
2724        {
2725            return Ok(vec![control_error_frame(
2726                &frame,
2727                "invalid_hello",
2728                "connection has open client routes and cannot also register as a module",
2729            )?]);
2730        }
2731
2732        // Kept for scope sync authority, which goes only to the connection that
2733        // presented the module's current launch nonce. Recorded before the
2734        // registration is attempted: a connection whose registration then fails
2735        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2736        self.hello_launch_nonces
2737            .lock()
2738            .unwrap_or_else(|poisoned| poisoned.into_inner())
2739            .record(connection_id, hello.launch_nonce.as_deref());
2740        let control_ops = effective_module_control_ops(hello.control_ops);
2741        // Built before anything is registered so an encoding failure leaves no
2742        // registry or forwarding state behind.
2743        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2744        if swap_candidate {
2745            return self.register_swap_candidate(
2746                connection_id,
2747                sink,
2748                &frame,
2749                hello.manifest,
2750                negotiated_ver,
2751                control_ops,
2752                hello_ack,
2753            );
2754        }
2755        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2756        // is refused, but `catalog.list` does not show the module and the
2757        // catalog generation does not move until it is published below, once
2758        // its forwarding endpoint exists. Listing it now would advertise a
2759        // module that `route.open` still refuses for having no forwarding
2760        // connection.
2761        let registration = match self
2762            .registry
2763            .register_unpublished_verified_with_control_ops(
2764                hello.manifest,
2765                negotiated_ver,
2766                connection_id,
2767                control_ops,
2768                nonce_verified,
2769            ) {
2770            Ok(registration) => registration,
2771            Err(RegistryError::DuplicateModuleId { module_id }) => {
2772                return Ok(vec![control_error_frame(
2773                    &frame,
2774                    "duplicate_module_id",
2775                    format!(
2776                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2777                    ),
2778                )?])
2779            }
2780            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2781                return Ok(vec![control_error_frame(
2782                    &frame,
2783                    "invalid_module_id",
2784                    err.to_string(),
2785                )?])
2786            }
2787            Err(err) => {
2788                return Ok(vec![control_error_frame(
2789                    &frame,
2790                    "registry_error",
2791                    err.to_string(),
2792                )?])
2793            }
2794        };
2795        #[cfg(test)]
2796        self.hold_between_registry_and_forwarding();
2797
2798        let reply = if let Some(sink) = sink {
2799            // The forwarding table's module store is also the daemon-to-module
2800            // control-RPC lane, so every HELLO gets a live endpoint even when the
2801            // manifest has no routable provider role. Non-routable modules still
2802            // cannot receive route.bind in production: `handle_route_open` checks
2803            // the registry manifest with `target_has_required_role` before the
2804            // only production call to `begin_route_bind_relay_for` below that
2805            // route.open path. The remaining direct relay callers are unit tests
2806            // and benchmark harnesses that construct forwarding state explicitly.
2807            //
2808            // The HELLO_ACK is queued by the forwarding table itself, before the
2809            // endpoint becomes visible, and is NOT returned as a reply. A module
2810            // reads HELLO_ACK first and exits on anything else; a reply is only
2811            // written after this handler returns, by which time a route.open on
2812            // another connection could already have queued a route.bind request
2813            // for this module ahead of it.
2814            let concurrency = manifest_concurrency(&registration.manifest);
2815            if let Err(err) = self.forwarding.register_module_connection_acked(
2816                connection_id,
2817                registration.manifest.module_id.clone(),
2818                negotiated_ver,
2819                concurrency,
2820                sink,
2821                hello_ack,
2822            ) {
2823                // Forwarding registration failed, so there is no forwarding
2824                // state to tear down. Remove the registry entry, which was never
2825                // published and so never listed or counted in the catalog
2826                // generation, and signal the release watch directly.
2827                if matches!(
2828                    self.deregister_connection(
2829                        connection_id,
2830                        RegistrationEndReason::RegistrationFailed,
2831                    ),
2832                    Ok(r) if !r.is_empty()
2833                ) {
2834                    crate::supervise::notify_registration_release();
2835                }
2836                return Ok(vec![control_error_frame(
2837                    &frame,
2838                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2839                        "invalid_hello"
2840                    } else {
2841                        forwarding_error_code(&err)
2842                    },
2843                    err.to_string(),
2844                )?]);
2845            }
2846            Vec::new()
2847        } else {
2848            // No sink means no forwarding endpoint, so nothing can be routed
2849            // ahead of the ack; it goes out as the reply.
2850            vec![hello_ack]
2851        };
2852
2853        // The module is routable now (or, without a sink, as routable as it
2854        // will ever be), so the catalog may list it. Publishing bumps the
2855        // catalog generation in the same registry critical section that makes
2856        // the module listed.
2857        if !self
2858            .registry
2859            .publish_registration(connection_id)
2860            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2861        {
2862            debug!(
2863                module_id = %registration.manifest.module_id,
2864                connection_id = connection_id.get(),
2865                "registration was removed before it could be published"
2866            );
2867        }
2868
2869        // Exposure over assumption: Concurrency's serde default is pinned to the
2870        // pre-field behavior (ModuleManaged), so a management surface that is
2871        // genuinely Serial and just never declared it inherits concurrent
2872        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2873        // turns "no module has been bitten yet" into the checkable claim "no
2874        // module is exposed" -- one read of the boot log instead of a fleet
2875        // audit. Detected from the raw HELLO bytes because the serde default
2876        // deliberately erases the absent/declared distinction from the type.
2877        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2878            info!(
2879                module_id = %registration.manifest.module_id,
2880                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2881            );
2882        }
2883
2884        self.apply_registration_capabilities(&registration);
2885
2886        self.supervisor
2887            .record_activity_registration(&registration.manifest.module_id);
2888
2889        info!(
2890            module_id = %registration.manifest.module_id,
2891            module_version = %registration.manifest.module_version,
2892            negotiated_ver,
2893            routable_provider = manifest_provides_routable_role(&registration.manifest),
2894            connection_id = connection_id.get(),
2895            "module registered"
2896        );
2897
2898        Ok(reply)
2899    }
2900
2901    /// Register a HELLO the swap gate admitted into the candidate slot of the
2902    /// registry and of forwarding, where it is reachable over its own
2903    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2904    /// nothing routes to it until the supervisor cuts over.
2905    ///
2906    /// Registry first (unpublished), then forwarding, then publication, the
2907    /// same order as an ordinary HELLO; a forwarding failure removes the
2908    /// registry entry again. The swap supervisor treats a published, ready
2909    /// candidate as registered and promotes it, and promotion needs the
2910    /// candidate's forwarding endpoint, so the candidate is published only
2911    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2912    /// recompute capability verdicts or run the capability deny census
2913    /// (`enforce_capability_denies`): both are about the module in the active
2914    /// slot, which `route.open` can target. A candidate sits in the candidate
2915    /// slot, so it neither provides capabilities to other modules nor can be
2916    /// routed to until promotion moves it into the active slot.
2917    #[allow(clippy::too_many_arguments)]
2918    fn register_swap_candidate(
2919        &self,
2920        connection_id: ConnectionId,
2921        sink: Option<crate::FrameSink>,
2922        frame: &Frame,
2923        manifest: ModuleManifest,
2924        negotiated_ver: u8,
2925        control_ops: Vec<String>,
2926        hello_ack: Frame,
2927    ) -> Result<Vec<Frame>, RouterError> {
2928        let module_id = manifest.module_id.clone();
2929        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2930            manifest,
2931            negotiated_ver,
2932            connection_id,
2933            control_ops,
2934        ) {
2935            Ok(registration) => registration,
2936            Err(RegistryError::DuplicateModuleId { module_id }) => {
2937                return Ok(vec![control_error_frame(
2938                    frame,
2939                    "duplicate_module_id",
2940                    format!(
2941                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2942                    ),
2943                )?])
2944            }
2945            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2946                return Ok(vec![control_error_frame(
2947                    frame,
2948                    "invalid_module_id",
2949                    err.to_string(),
2950                )?])
2951            }
2952            Err(err) => {
2953                return Ok(vec![control_error_frame(
2954                    frame,
2955                    "registry_error",
2956                    err.to_string(),
2957                )?])
2958            }
2959        };
2960        #[cfg(test)]
2961        self.hold_between_registry_and_forwarding();
2962        let reply = if let Some(sink) = sink {
2963            // Same ordering as an ordinary HELLO: the forwarding table queues
2964            // the HELLO_ACK before the candidate endpoint is inserted, because
2965            // a module exits if its first frame after HELLO is anything else.
2966            let concurrency = manifest_concurrency(&registration.manifest);
2967            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2968                connection_id,
2969                module_id.clone(),
2970                negotiated_ver,
2971                concurrency,
2972                sink,
2973                hello_ack,
2974            ) {
2975                if matches!(
2976                    self.deregister_connection(
2977                        connection_id,
2978                        RegistrationEndReason::RegistrationFailed,
2979                    ),
2980                    Ok(r) if !r.is_empty()
2981                ) {
2982                    crate::supervise::notify_registration_release();
2983                }
2984                return Ok(vec![control_error_frame(
2985                    frame,
2986                    forwarding_error_code(&err),
2987                    err.to_string(),
2988                )?]);
2989            }
2990            Vec::new()
2991        } else {
2992            vec![hello_ack]
2993        };
2994        if !self
2995            .registry
2996            .publish_registration(connection_id)
2997            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998        {
2999            debug!(
3000                module_id = %module_id,
3001                connection_id = connection_id.get(),
3002                "swap candidate registration was removed before it could be published"
3003            );
3004        }
3005        self.supervisor.mark_swap_candidate_admitted(&module_id);
3006        info!(
3007            module_id = %module_id,
3008            module_version = %registration.manifest.module_version,
3009            negotiated_ver,
3010            ready = registration.ready,
3011            connection_id = connection_id.get(),
3012            "swap candidate registered; not routable until cutover"
3013        );
3014        Ok(reply)
3015    }
3016
3017    fn build_hello_ack(
3018        &self,
3019        frame: &Frame,
3020        negotiated_ver: u8,
3021        module_id: &str,
3022    ) -> Result<Frame, RouterError> {
3023        let ack = ModuleHelloAckBody {
3024            negotiated_ver,
3025            subc_ops: module_subc_ops(),
3026            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3027            storage: self
3028                .storage_config
3029                .as_ref()
3030                .map(|cfg| cfg.descriptor_for(module_id)),
3031            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3032        };
3033        let body = serde_json::to_vec(&ack).map_err(|err| {
3034            RouterError::backend(
3035                0,
3036                frame.header.corr,
3037                format!("failed to encode HELLO_ACK: {err}"),
3038            )
3039        })?;
3040
3041        Frame::build_with_version(
3042            negotiated_ver,
3043            FrameType::HelloAck,
3044            control_flags(),
3045            0,
3046            0,
3047            frame.header.corr,
3048            body,
3049        )
3050        .map_err(RouterError::FrameBuild)
3051    }
3052
3053    async fn handle_client_control_request(
3054        &self,
3055        ctx: &RouteCtx,
3056        frame: Frame,
3057        request: ClientControlRequest,
3058        route_open_timings: &mut Option<RouteOpenTimings>,
3059    ) -> Result<Vec<Frame>, RouterError> {
3060        if let Some(operation) = privileged_operation(&request) {
3061            let caller = self.resolve_caller(ctx.connection_id).await;
3062            let op = client_control_request_op(&request);
3063            let text = prompt_text(operation, &caller);
3064            match self.privileged_operator_prompt {
3065                PrivilegedOperatorPrompt::Log => {
3066                    info!(target: "subc_daemon::privileged_prompt", operation = op,
3067                        target = ?operation, caller = %caller.description(), summary = %text,
3068                        "privileged request would prompt the operator");
3069                }
3070                PrivilegedOperatorPrompt::Enforce => {
3071                    let key =
3072                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3073                    let stamp = self
3074                        .operator_apps
3075                        .lock()
3076                        .unwrap_or_else(|p| p.into_inner())
3077                        .get(&ctx.connection_id)
3078                        .cloned();
3079                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3080                        &self.operator_app_policy,
3081                        &self.launch_table,
3082                        stamp.as_ref(),
3083                    )
3084                    .then(|| self.shortcut_context(stamp, None, None));
3085                    let outcome = self
3086                        .forwarding
3087                        .operator_confirms()
3088                        .approve_privileged_with_shortcut(
3089                            ctx,
3090                            &frame,
3091                            op,
3092                            text,
3093                            caller.description(),
3094                            key,
3095                            shortcut,
3096                        )
3097                        .await;
3098                    if outcome.code != "confirmed" {
3099                        return Ok(vec![outcome.frame(&frame)]);
3100                    }
3101                }
3102            }
3103        }
3104        match request {
3105            ClientControlRequest::DaemonShutdownIntent { intent } => {
3106                if intent.as_str() != Some("restart") {
3107                    return Ok(vec![control_error_frame(
3108                        &frame,
3109                        "invalid_params",
3110                        "intent must be 'restart'",
3111                    )?]);
3112                }
3113                let response = ClientControlResponse::DaemonShutdownIntent {
3114                    intent: "restart".to_owned(),
3115                    expires_at_ms: self.supervisor.announce_daemon_restart(),
3116                };
3117                Ok(vec![control_response_body_frame(
3118                    &frame,
3119                    &response,
3120                    "ClientControlResponse::DaemonShutdownIntent",
3121                )?])
3122            }
3123            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3124            ClientControlRequest::CatalogList { module_id } => {
3125                self.handle_catalog_list(frame, module_id)
3126            }
3127            ClientControlRequest::RouteOpen {
3128                target,
3129                identity,
3130                consumer_identity,
3131                consumer_capabilities,
3132                role_versions,
3133                busy_refusals,
3134                admission_facts,
3135                scope,
3136                ..
3137            } => {
3138                self.handle_route_open(
3139                    ctx,
3140                    frame,
3141                    RouteOpenRequest {
3142                        target,
3143                        identity,
3144                        consumer_identity,
3145                        consumer_capabilities,
3146                        role_versions,
3147                        busy_refusals,
3148                        admission_facts,
3149                        scope,
3150                    },
3151                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3152                )
3153                .await
3154            }
3155            ClientControlRequest::RoutePoll {
3156                route_channel,
3157                route_epoch,
3158                kind,
3159            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3160            ClientControlRequest::SupervisorList {} => {
3161                // Filesystem-backed diagnostics must not hold up this connection's reader.
3162                let handler = self.clone();
3163                let sink = ctx.egress.clone();
3164                tokio::spawn(
3165                    async move {
3166                        let result = handler.handle_supervisor_list(frame.clone()).await;
3167                        let replies = match result {
3168                            Ok(replies) => replies,
3169                            Err(error) => {
3170                                control_error_frame(&frame, "backend_error", error.to_string())
3171                                    .into_iter()
3172                                    .collect()
3173                            }
3174                        };
3175                        for reply in replies {
3176                            if sink.send(reply).await.is_err() {
3177                                break;
3178                            }
3179                        }
3180                    }
3181                    .with_current_subscriber(),
3182                );
3183                Ok(Vec::new())
3184            }
3185            ClientControlRequest::SupervisorSpawnSnapshot {} => {
3186                self.handle_supervisor_spawn_snapshot(frame)
3187            }
3188            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3189                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3190            }
3191            ClientControlRequest::SupervisorRestart {
3192                module_id,
3193                drain_timeout_ms,
3194            } => {
3195                self.log_supervisor_request_received(
3196                    ctx,
3197                    frame.header.corr,
3198                    ops::SUPERVISOR_RESTART,
3199                    Some(&module_id),
3200                    None,
3201                )?;
3202                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3203                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3204                    .await
3205            }
3206            ClientControlRequest::SupervisorSwap {
3207                module_id,
3208                ready_timeout_ms,
3209            } => {
3210                self.log_supervisor_request_received(
3211                    ctx,
3212                    frame.header.corr,
3213                    ops::SUPERVISOR_SWAP,
3214                    Some(&module_id),
3215                    None,
3216                )?;
3217                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3218                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3219                    .await
3220            }
3221            ClientControlRequest::SupervisorReload { module_id } => {
3222                self.log_supervisor_request_received(
3223                    ctx,
3224                    frame.header.corr,
3225                    ops::SUPERVISOR_RELOAD,
3226                    Some(&module_id),
3227                    None,
3228                )?;
3229                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3230                self.handle_supervisor_reload(frame, module_id, requester)
3231                    .await
3232            }
3233            ClientControlRequest::SupervisorRescan { preview } => {
3234                if !preview {
3235                    self.log_supervisor_request_received(
3236                        ctx,
3237                        frame.header.corr,
3238                        ops::SUPERVISOR_RESCAN,
3239                        None,
3240                        None,
3241                    )?;
3242                }
3243                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3244                let operation = self.handle_supervisor_rescan(
3245                    frame,
3246                    if preview {
3247                        RescanMode::Preview
3248                    } else {
3249                        RescanMode::Execute
3250                    },
3251                    requester,
3252                );
3253                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3254                    .await
3255            }
3256            ClientControlRequest::SupervisorRescanChecked {
3257                preview,
3258                candidate,
3259                expect,
3260            } => {
3261                self.observe_control_wait(
3262                    ctx,
3263                    None,
3264                    ops::SUPERVISOR_RESCAN,
3265                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3266                )
3267                .await
3268            }
3269            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3270                self.log_supervisor_request_received(
3271                    ctx,
3272                    frame.header.corr,
3273                    ops::SUPERVISOR_RELEASE_RESERVED,
3274                    Some(&module_id),
3275                    None,
3276                )?;
3277                self.handle_supervisor_release_reserved(frame, module_id)
3278                    .await
3279            }
3280            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3281                self.log_supervisor_request_received(
3282                    ctx,
3283                    frame.header.corr,
3284                    ops::SUPERVISOR_SET_ENABLED,
3285                    Some(&module_id),
3286                    Some(enabled),
3287                )?;
3288                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3289                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3290                    .await
3291            }
3292            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3293                self.handle_supervisor_health_probe(ctx, frame, module_id)
3294                    .await
3295            }
3296            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3297            ClientControlRequest::SupervisorRoutes { module_id } => {
3298                self.handle_supervisor_routes(frame, module_id)
3299            }
3300            ClientControlRequest::SupervisorProvenance { module_id } => {
3301                self.handle_supervisor_provenance(ctx, frame, module_id)
3302                    .await
3303            }
3304            ClientControlRequest::SupervisorStderrTail {
3305                module_id,
3306                max_lines,
3307                max_bytes,
3308            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3309            ClientControlRequest::SupervisorTerminals { module_id } => {
3310                self.handle_supervisor_terminals(ctx, frame, module_id)
3311                    .await
3312            }
3313        }
3314    }
3315
3316    fn log_supervisor_request_received(
3317        &self,
3318        ctx: &RouteCtx,
3319        corr: u64,
3320        op: &'static str,
3321        module_id: Option<&str>,
3322        enabled: Option<bool>,
3323    ) -> Result<(), RouterError> {
3324        let caller = self.supervisor_requester(ctx, corr)?;
3325        let caller = principal_label(&caller);
3326        // `Option` fields are recorded only when present, so a request with no
3327        // module id (rescan) or no enabled flag simply omits that field.
3328        info!(
3329            target: "control",
3330            op,
3331            module_id,
3332            enabled,
3333            connection_id = ctx.connection_id.get(),
3334            caller = %caller,
3335            "supervisor request received"
3336        );
3337        Ok(())
3338    }
3339
3340    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3341        Ok(self
3342            .registry
3343            .get_module_by_connection(ctx.connection_id)
3344            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3345            .map(|registration| Principal::Reserved {
3346                module_id: registration.manifest.module_id,
3347            })
3348            .unwrap_or(Principal::Direct))
3349    }
3350
3351    fn handle_module_control_request(
3352        &self,
3353        connection_id: ConnectionId,
3354        frame: Frame,
3355        request: ModuleControlRequestFromModule,
3356    ) -> Result<Vec<Frame>, RouterError> {
3357        match request {
3358            ModuleControlRequestFromModule::CatalogUpdate {
3359                provides,
3360                capabilities,
3361                ready,
3362            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3363            ModuleControlRequestFromModule::LiveRoots {} => {
3364                let registered = self
3365                    .registry
3366                    .get_module_by_connection(connection_id)
3367                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3368                let Some(registration) = registered else {
3369                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3370                };
3371                let response = self
3372                    .forwarding
3373                    .live_roots(&registration.manifest.module_id)
3374                    .map_err(RouterError::Forwarding)?;
3375                Ok(vec![control_response_body_frame(
3376                    &frame,
3377                    &response,
3378                    "ModuleControlResponseToModule::LiveRoots",
3379                )?])
3380            }
3381            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3382                self.handle_scope_sync(connection_id, frame, generation, scopes)
3383            }
3384            ModuleControlRequestFromModule::ScopeApply {
3385                generation,
3386                upsert,
3387                end,
3388            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3389            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3390                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3391            }
3392        }
3393    }
3394
3395    fn handle_scope_served_by(
3396        &self,
3397        connection_id: ConnectionId,
3398        frame: Frame,
3399    ) -> Result<Vec<Frame>, RouterError> {
3400        let registered = self
3401            .registry
3402            .get_module_by_connection(connection_id)
3403            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3404        let Some(registered) = registered else {
3405            return Ok(vec![control_error_frame(
3406                &frame,
3407                "not_registered",
3408                "scope.served_by requires an active module registration owned by this connection",
3409            )?]);
3410        };
3411        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3412        {
3413            Ok(request) => request,
3414            Err(err) => {
3415                return Ok(vec![control_error_frame(
3416                    &frame,
3417                    "invalid_control_body",
3418                    format!("malformed scope.served_by body: {err}"),
3419                )?])
3420            }
3421        };
3422        let answer = self
3423            .scopes
3424            .read()
3425            .unwrap_or_else(|poisoned| poisoned.into_inner())
3426            .served_by(
3427                &registered.manifest.module_id,
3428                &request.scope_ref,
3429                request.scope_epoch,
3430                &request.module_id,
3431            );
3432        let reply = subc_control::ScopeServedByReply::new(
3433            answer,
3434            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3435        );
3436        Ok(vec![control_response_body_frame(
3437            &frame,
3438            &reply,
3439            "ScopeServedByReply",
3440        )?])
3441    }
3442
3443    fn handle_operator_confirm(
3444        &self,
3445        ctx: &RouteCtx,
3446        frame: Frame,
3447    ) -> Result<Vec<Frame>, RouterError> {
3448        use crate::operator_confirm::{audit, Outcome};
3449        let registration = self
3450            .registry
3451            .get_module_by_connection(ctx.connection_id)
3452            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3453        let Some(registration) = registration else {
3454            let outcome = Outcome::refusal("not_registered");
3455            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3456            return Ok(vec![outcome.frame(&frame)]);
3457        };
3458        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3459            Ok(request) => request,
3460            Err(_) => {
3461                let outcome = Outcome::refusal("invalid_control_body");
3462                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3463                return Ok(vec![outcome.frame(&frame)]);
3464            }
3465        };
3466        let declared = matches!(
3467            request.kind,
3468            Some(subc_protocol::session::OperatorConfirmKind::Operation)
3469        ) && request.operation.as_ref().is_some_and(|operation| {
3470            registration
3471                .manifest
3472                .confirm_operations
3473                .as_ref()
3474                .is_some_and(|operations| operations.contains(operation))
3475        });
3476        let module_id = registration.manifest.module_id;
3477        // Read the launch nonce (under its own lock) before taking the forwarding
3478        // table's lock below: holding forwarding while waiting on another daemon
3479        // lock risks a lock-order deadlock with paths that take them the other way.
3480        let nonce = self
3481            .hello_launch_nonces
3482            .lock()
3483            .unwrap_or_else(|p| p.into_inner())
3484            .nonce(ctx.connection_id)
3485            .map(str::to_owned);
3486        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3487            self.supervisor
3488                .spawned_consumer_authorized(&module_id, nonce)
3489        });
3490        let confirms = self.forwarding.operator_confirms();
3491        self.forwarding
3492            .with_operator_route(
3493                ctx.connection_id,
3494                request.route_channel,
3495                request.route_epoch,
3496                |binding| {
3497                    let shortcut = binding
3498                        .filter(|binding| {
3499                            nonce_proven
3500                                && crate::operator_confirm::valid_summary(&request.summary)
3501                                && declared
3502                                && crate::operator_confirm::ShortcutContext::route_eligible(
3503                                    &self.operator_app_policy,
3504                                    &self.launch_table,
3505                                    binding,
3506                                )
3507                        })
3508                        .map(|binding| {
3509                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3510                                &self.operator_app_policy,
3511                                binding,
3512                            );
3513                            let stamp = if phone.is_some() {
3514                                None
3515                            } else {
3516                                binding.operator_app.clone()
3517                            };
3518                            self.shortcut_context(
3519                                stamp,
3520                                phone,
3521                                Some((request.route_channel, request.route_epoch)),
3522                            )
3523                        });
3524                    confirms.admit_with_shortcut(
3525                        ctx,
3526                        frame,
3527                        module_id,
3528                        nonce_proven,
3529                        request,
3530                        binding,
3531                        shortcut,
3532                    )
3533                },
3534            )
3535            .map_err(RouterError::Forwarding)
3536    }
3537
3538    fn shortcut_context(
3539        &self,
3540        stamp: Option<crate::operator_apps::OperatorAppStamp>,
3541        phone_fingerprint: Option<String>,
3542        route: Option<(u16, u32)>,
3543    ) -> crate::operator_confirm::ShortcutContext {
3544        crate::operator_confirm::ShortcutContext {
3545            ledger: self.activity.clone(),
3546            forwarding: self.forwarding.clone(),
3547            launches: self.launch_table.clone(),
3548            stamp,
3549            phone_fingerprint,
3550            route,
3551            connections: self.operator_apps.clone(),
3552            #[cfg(test)]
3553            hooks: Default::default(),
3554        }
3555    }
3556
3557    /// Explicit test-only phone policy; daemon configuration cannot select it.
3558    #[cfg(any(test, feature = "test-support"))]
3559    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3560        self.operator_app_policy.phone_enabled = enabled;
3561        self
3562    }
3563
3564    /// Resolve folders before taking the scope-table lock: a slow filesystem
3565    /// must not hold up admissions for unrelated scoped routes.
3566    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3567        let mut scopes = ScopeRecords::from(scopes);
3568        for (index, record) in scopes.records.iter_mut().enumerate() {
3569            if let Some(refusal) = Self::canonicalize_workdir(record) {
3570                scopes.refusals.insert(index, refusal);
3571            }
3572        }
3573        scopes
3574    }
3575
3576    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3577        let workdir = record.workdir.as_mut()?;
3578        let path = std::path::Path::new(workdir);
3579        if !path.is_absolute() {
3580            return Some((
3581                error_codes::INVALID_WORKDIR,
3582                "workdir must be absolute".to_string(),
3583            ));
3584        }
3585        match ProjectRootId::from_path_allowing_missing(path) {
3586            Ok(root) => match root.as_path().to_str() {
3587                Some(canonical) => {
3588                    *workdir = canonical.to_string();
3589                    None
3590                }
3591                None => Some((
3592                    error_codes::INVALID_WORKDIR,
3593                    "canonical workdir is not UTF-8".to_string(),
3594                )),
3595            },
3596            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3597        }
3598    }
3599
3600    /// `scope.sync`: the owner is the module registered on this connection.
3601    /// A connection with no registration (every client connection, `direct`
3602    /// included) is refused `not_registered` before the table is consulted.
3603    fn handle_scope_sync(
3604        &self,
3605        connection_id: ConnectionId,
3606        frame: Frame,
3607        generation: u64,
3608        scopes: Vec<ScopeRecord>,
3609    ) -> Result<Vec<Frame>, RouterError> {
3610        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3611    }
3612
3613    fn handle_scope_change(
3614        &self,
3615        connection_id: ConnectionId,
3616        frame: Frame,
3617        generation: u64,
3618        scopes: Vec<ScopeRecord>,
3619        end: Option<Vec<ScopeEnd>>,
3620    ) -> Result<Vec<Frame>, RouterError> {
3621        let is_apply = end.is_some();
3622        let Some(registration) = self
3623            .registry
3624            .get_module_by_connection(connection_id)
3625            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3626        else {
3627            return Ok(vec![control_error_frame(
3628                &frame,
3629                "not_registered",
3630                "scope.sync and scope.apply require an active module registration owned by this connection",
3631            )?]);
3632        };
3633        let owner = registration.manifest.module_id;
3634        let scopes = Self::prepare_scope_records(scopes);
3635        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3636        let is_current_launch = |connection: ConnectionId| {
3637            self.hello_launch_nonces
3638                .lock()
3639                .unwrap_or_else(|poisoned| poisoned.into_inner())
3640                .presented(connection, current_nonce.as_deref())
3641        };
3642        // Lock order is the scope table, then the forwarding table: the new
3643        // tags are published, and the routes the change closes are selected,
3644        // while the scope table is still write-locked, so no admission can read
3645        // a record whose tag is not yet published.
3646        let mut table = self
3647            .scopes
3648            .write()
3649            .unwrap_or_else(|poisoned| poisoned.into_inner());
3650        let outcome = match end {
3651            Some(end) => table.apply(
3652                &owner,
3653                connection_id,
3654                is_current_launch,
3655                generation,
3656                scopes,
3657                end,
3658            ),
3659            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3660        };
3661        let drained = match &outcome {
3662            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3663            Err(_) => Vec::new(),
3664        };
3665        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3666        drop(table);
3667        if has_deadlines {
3668            self.scope_deadline_added.notify_one();
3669        }
3670        match outcome {
3671            Ok(applied) => {
3672                let counts = ScopeOutcomeCounts::of(&applied.results);
3673                info!(
3674                    owner = %owner,
3675                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3676                    generation,
3677                    records = applied.results.len(),
3678                    created = counts.created,
3679                    replaced = counts.replaced,
3680                    updated = counts.updated,
3681                    unchanged = counts.unchanged,
3682                    refused = counts.refused,
3683                    ended = applied.ended.len(),
3684                    tag_changes = applied.tag_changes.len(),
3685                    routes_closed = drained.len(),
3686                    "scope change accepted"
3687                );
3688                // An accepted scope change can still refuse individual records, and the
3689                // owner is the only party that sees the reply. Name them here so
3690                // an operator can tell a refused session from a missing one
3691                // without the owner's logs. Capped so a call that refuses
3692                // thousands cannot flood the log; the count above is complete.
3693                for refused in applied
3694                    .results
3695                    .iter()
3696                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3697                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3698                {
3699                    warn!(
3700                        owner = %owner,
3701                        generation,
3702                        scope_ref = %refused.scope_ref,
3703                        scope_epoch = refused.scope_epoch,
3704                        code = refused.code.as_deref().unwrap_or(""),
3705                        "scope record refused"
3706                    );
3707                }
3708                self.close_scope_drained_routes(drained);
3709                let response = if is_apply {
3710                    ModuleControlResponseToModule::ScopeApply {
3711                        generation,
3712                        results: applied.results,
3713                        end_results: applied.end_results,
3714                        ended: applied.ended,
3715                    }
3716                } else {
3717                    ModuleControlResponseToModule::ScopeSync {
3718                        generation,
3719                        results: applied.results,
3720                        ended: applied.ended,
3721                    }
3722                };
3723                Ok(vec![control_response_body_frame(
3724                    &frame,
3725                    &response,
3726                    "scope change response",
3727                )?])
3728            }
3729            Err(refusal) => {
3730                info!(
3731                    owner = %owner,
3732                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3733                    generation,
3734                    code = refusal.code,
3735                    "scope change refused"
3736                );
3737                Ok(vec![control_error_frame(
3738                    &frame,
3739                    refusal.code,
3740                    refusal.message,
3741                )?])
3742            }
3743        }
3744    }
3745
3746    /// Tell both ends of each route a scope change closed. The module gets a
3747    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3748    /// the client's route handle. The client also gets `route.closed` with the
3749    /// scope reason, one push per module and reason, so it can tell a revoked
3750    /// route from an ordinary close and not reopen it.
3751    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3752        if drained.is_empty() {
3753            return;
3754        }
3755        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3756            BTreeMap::new();
3757        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3758        for route in drained {
3759            warn!(
3760                module_id = %route.module_id,
3761                reason = ?route.reason,
3762                client_connection_id = route.client.connection_id.get(),
3763                route_channel = route.client.channel,
3764                "closing route because its scope changed"
3765            );
3766            pushes
3767                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3768                .or_insert_with(|| (route.reason, Vec::new()))
3769                .1
3770                .push(EndpointRoute {
3771                    goodbye_target: route.client.clone(),
3772                    principal: Principal::Unverified,
3773                    bound_at: Instant::now(),
3774                    draining: false,
3775                    drain_reason: None,
3776                });
3777            goodbyes.push(route.module);
3778            goodbyes.push(route.client);
3779        }
3780        for ((module_id, _), (reason, routes)) in pushes {
3781            send_route_control_pushes(
3782                &self.forwarding,
3783                routes,
3784                ClientControlPush::RouteClosed {
3785                    module_id,
3786                    channels: Vec::new(),
3787                    reason,
3788                    drained: false,
3789                    abandoned: 0,
3790                    excluded_subscriptions: 0,
3791                    terminal: Some(false),
3792                },
3793            );
3794        }
3795        self.emit_route_goodbyes(goodbyes);
3796    }
3797
3798    /// `scope.describe`: any registered module may read any scope, because a
3799    /// provider must read the scope a route it serves is stamped with.
3800    fn handle_scope_describe(
3801        &self,
3802        connection_id: ConnectionId,
3803        frame: Frame,
3804        owner: Principal,
3805        scope_ref: String,
3806    ) -> Result<Vec<Frame>, RouterError> {
3807        let registered = self
3808            .registry
3809            .get_module_by_connection(connection_id)
3810            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3811        let Some(registered) = registered else {
3812            return Ok(vec![control_error_frame(
3813                &frame,
3814                "not_registered",
3815                "scope.describe requires an active module registration owned by this connection",
3816            )?]);
3817        };
3818        let mut description = self
3819            .scopes
3820            .read()
3821            .unwrap_or_else(|poisoned| poisoned.into_inner())
3822            .describe(&owner, &scope_ref);
3823        // Describe readers also decode strict ScopeAttributes. Attribution is
3824        // safe to omit unless the requesting module declares it understands it.
3825        if !registered
3826            .manifest
3827            .capabilities
3828            .as_ref()
3829            .is_some_and(|caps| {
3830                caps.provides
3831                    .iter()
3832                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3833            })
3834        {
3835            if let Some(stamp) = description.stamp.as_mut() {
3836                stamp.attributes.on_behalf_of_agent = None;
3837            }
3838        }
3839        if !registered
3840            .manifest
3841            .capabilities
3842            .as_ref()
3843            .is_some_and(|caps| {
3844                caps.provides
3845                    .iter()
3846                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3847            })
3848        {
3849            if let Some(stamp) = description.stamp.as_mut() {
3850                stamp.attributes.flow_version = None;
3851                stamp.attributes.flow_code_hash = None;
3852                stamp.attributes.flow_install_id = None;
3853            }
3854        }
3855        let owner_configured = match &owner {
3856            // Ask whether the owner is configured (`is_configured`), not
3857            // whether it is on the roster (`get(..).is_some()`): a supervised
3858            // module's process can register and describe a scope before the
3859            // supervisor has put it on the roster.
3860            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3861            _ => false,
3862        };
3863        let response = ModuleControlResponseToModule::ScopeDescribe {
3864            status: description.status,
3865            scope_epoch: description.scope_epoch,
3866            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3867            owner_synced: description.owner_synced,
3868            owner_configured,
3869            scope: description.stamp,
3870        };
3871        Ok(vec![control_response_body_frame(
3872            &frame,
3873            &response,
3874            "ModuleControlResponseToModule::ScopeDescribe",
3875        )?])
3876    }
3877
3878    fn handle_catalog_update(
3879        &self,
3880        connection_id: ConnectionId,
3881        frame: Frame,
3882        provides: Vec<ProviderRole>,
3883        capabilities: Option<CapabilityDeclarations>,
3884        ready: Option<bool>,
3885    ) -> Result<Vec<Frame>, RouterError> {
3886        self.refresh_capability_requirements();
3887        let Some(registration) = self
3888            .registry
3889            .get_module_by_connection(connection_id)
3890            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3891        else {
3892            return Ok(vec![control_error_frame(
3893                &frame,
3894                "not_registered",
3895                "catalog.update requires an active module registration owned by this connection",
3896            )?]);
3897        };
3898
3899        if let Some(message) =
3900            catalog_update_frozen_field_message(&registration.manifest, &provides)
3901        {
3902            return Ok(vec![control_error_frame(
3903                &frame,
3904                "catalog_update_frozen_field",
3905                message,
3906            )?]);
3907        }
3908
3909        let mut candidate = registration.manifest.clone();
3910        candidate.provides = provides.clone();
3911        candidate.capabilities = capabilities
3912            .clone()
3913            .or_else(|| registration.manifest.capabilities.clone());
3914        if let Some(rich_ui) = &candidate.rich_ui {
3915            if let Err(err) = rich_ui.validate() {
3916                return Ok(vec![control_error_frame(
3917                    &frame,
3918                    "invalid_manifest",
3919                    err.to_string(),
3920                )?]);
3921            }
3922        }
3923        if let Err(err) = candidate.validate_capability_grammar() {
3924            return Ok(vec![control_error_frame(
3925                &frame,
3926                "invalid_capability_grammar",
3927                err.to_string(),
3928            )?]);
3929        }
3930
3931        // Updates must honor the same reserved owner as initial registration;
3932        // otherwise an empty HELLO could acquire the claim after admission.
3933        let mut conflicts = self
3934            .capability_evaluator
3935            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3936        if let Some(conflict) = conflicts.first() {
3937            let message = format!(
3938                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3939                conflict.capability, conflict.claimants[0], candidate.module_id
3940            );
3941            for conflict in &mut conflicts {
3942                conflict.source = DuplicateClaimSource::CatalogUpdate;
3943            }
3944            log_duplicate_claim_events(conflicts);
3945            return Ok(vec![control_error_frame(
3946                &frame,
3947                "reserved_capability",
3948                message,
3949            )?]);
3950        }
3951
3952        let updated = self
3953            .registry
3954            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3955            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3956        if updated.is_none() {
3957            return Ok(vec![control_error_frame(
3958                &frame,
3959                "not_registered",
3960                "catalog.update requires an active module registration owned by this connection",
3961            )?]);
3962        }
3963        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3964            log_duplicate_claim_events(
3965                self.capability_evaluator
3966                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3967            );
3968        }
3969        if capability_census_trigger(
3970            registration.manifest.capabilities.as_ref(),
3971            updated
3972                .as_ref()
3973                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3974        ) {
3975            self.enforce_capability_denies();
3976        }
3977        self.refresh_capability_requirements();
3978
3979        let response = ModuleControlResponseToModule::CatalogUpdate {};
3980        control_response_body_frame(
3981            &frame,
3982            &response,
3983            "ModuleControlResponseToModule::CatalogUpdate",
3984        )
3985        .map(|frame| vec![frame])
3986    }
3987
3988    pub(crate) fn with_activity_ledger_config(
3989        mut self,
3990        run_dir: &Path,
3991        incarnation: String,
3992        config: crate::activity::ActivityConfig,
3993    ) -> Self {
3994        self.activity = Some(crate::activity::ActivityLedger::open_config(
3995            run_dir.join("activity.jsonl"),
3996            incarnation,
3997            config,
3998        ));
3999        self.supervisor.set_activity(self.activity.clone());
4000        self
4001    }
4002
4003    pub(crate) async fn configure_activity(
4004        &self,
4005        config: crate::activity::ActivityConfig,
4006    ) -> Result<(), &'static str> {
4007        match &self.activity {
4008            Some(ledger) => ledger.configure(config).await,
4009            None => Ok(()),
4010        }
4011    }
4012
4013    fn cancel_activity_subscription(
4014        &self,
4015        ctx: &RouteCtx,
4016        frame: &Frame,
4017    ) -> Result<bool, RouterError> {
4018        let task = self
4019            .activity_streams
4020            .lock()
4021            .unwrap_or_else(|p| p.into_inner())
4022            .remove(&(ctx.connection_id, frame.header.corr));
4023        let Some(task) = task else {
4024            return Ok(false);
4025        };
4026        task.abort();
4027        let end = Frame::build_with_version(
4028            response_version(frame),
4029            FrameType::StreamEnd,
4030            control_flags(),
4031            0,
4032            0,
4033            frame.header.corr,
4034            Vec::new(),
4035        )
4036        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4037        let sink = ctx.egress.clone();
4038        tokio::spawn(async move {
4039            let _ = sink.send(end).await;
4040        });
4041        Ok(true)
4042    }
4043
4044    async fn handle_activity_read(
4045        &self,
4046        ctx: &RouteCtx,
4047        frame: Frame,
4048    ) -> Result<Vec<Frame>, RouterError> {
4049        use subc_control::{
4050            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4051            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4052        };
4053        #[derive(Deserialize)]
4054        struct Identity {
4055            #[serde(default)]
4056            consumer_identity: Option<ConsumerIdentity>,
4057        }
4058        // A registered module connection already identifies its caller. Ignore
4059        // consumer_identity there so a module cannot borrow a mirror's identity.
4060        // Only separate consumer connections use the Direct/nonce trust boundary.
4061        let principal = match self
4062            .forwarding
4063            .module_id_for_connection(ctx.connection_id)
4064            .map_err(RouterError::Forwarding)?
4065        {
4066            Some(module_id) => Ok(Principal::Reserved { module_id }),
4067            None => {
4068                let identity: Identity = match serde_json::from_slice(&frame.body) {
4069                    Ok(identity) => identity,
4070                    Err(_) => {
4071                        return Ok(vec![control_error_frame(
4072                            &frame,
4073                            ACTIVITY_MALFORMED,
4074                            "malformed activity read",
4075                        )?])
4076                    }
4077                };
4078                self.route_open_principal(&frame, identity.consumer_identity)?
4079            }
4080        };
4081        let admitted = matches!(principal, Ok(Principal::Direct))
4082            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4083        if !admitted {
4084            return Ok(vec![control_error_frame(
4085                &frame,
4086                ACTIVITY_READ_FORBIDDEN,
4087                "activity reads require an operator or trusted mirror principal",
4088            )?]);
4089        }
4090        let Some(ledger) = &self.activity else {
4091            return Ok(vec![control_error_frame(
4092                &frame,
4093                ACTIVITY_UNAVAILABLE,
4094                "activity ledger unavailable",
4095            )?]);
4096        };
4097        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4098            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4099        {
4100            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4101                Ok(request) => ledger.query(request).await,
4102                Err(_) => Err(ACTIVITY_MALFORMED),
4103            };
4104            return match result {
4105                Ok(reply) => Ok(vec![control_response_body_frame(
4106                    &frame,
4107                    &reply,
4108                    ops::ACTIVITY_QUERY,
4109                )?]),
4110                Err(code) => Ok(vec![control_error_frame(
4111                    &frame,
4112                    code,
4113                    "activity query refused",
4114                )?]),
4115            };
4116        }
4117        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4118            Ok(request) => request,
4119            Err(_) => {
4120                return Ok(vec![control_error_frame(
4121                    &frame,
4122                    ACTIVITY_MALFORMED,
4123                    "malformed activity subscription",
4124                )?])
4125            }
4126        };
4127        let mut subscription = match ledger.subscribe(request.since).await {
4128            Ok(subscription) => subscription,
4129            Err(code) => {
4130                return Ok(vec![control_error_frame(
4131                    &frame,
4132                    code,
4133                    "activity subscription refused",
4134                )?])
4135            }
4136        };
4137        let key = (ctx.connection_id, frame.header.corr);
4138        let sink = ctx.egress.clone();
4139        let mut streams = self
4140            .activity_streams
4141            .lock()
4142            .unwrap_or_else(|p| p.into_inner());
4143        streams.retain(|_, task| !task.is_finished());
4144        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4145            return Ok(vec![control_error_frame(
4146                &frame,
4147                "duplicate_subscription",
4148                "activity subscription correlation id already in use",
4149            )?]);
4150        }
4151        let task = tokio::spawn(async move {
4152            loop {
4153                let entry = if let Some(entry) = subscription.replay.pop_front() {
4154                    Some(entry)
4155                } else {
4156                    subscription.receiver.recv().await
4157                };
4158                let Some(entry) = entry else {
4159                    break;
4160                };
4161                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4162                    return;
4163                };
4164                let Ok(next) = Frame::build_with_version(
4165                    response_version(&frame),
4166                    FrameType::StreamData,
4167                    control_flags(),
4168                    0,
4169                    0,
4170                    frame.header.corr,
4171                    body,
4172                ) else {
4173                    return;
4174                };
4175                if sink.send(next).await.is_err() {
4176                    return;
4177                }
4178            }
4179            // Dropping a full subscriber's sender drains its accepted entries
4180            // before the terminal error, preserving their order on the wire.
4181            if let Some(code) = subscription.terminal_code() {
4182                if let Ok(end) =
4183                    control_error_frame(&frame, code, "activity subscriber fell behind")
4184                {
4185                    let _ = sink.send(end).await;
4186                }
4187            }
4188        });
4189        streams.insert(key, task.abort_handle());
4190        Ok(Vec::new())
4191    }
4192
4193    async fn handle_activity_record(
4194        &self,
4195        ctx: &RouteCtx,
4196        frame: Frame,
4197    ) -> Result<Vec<Frame>, RouterError> {
4198        use subc_control::{
4199            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4200            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4201        };
4202        let result = async {
4203            let module = self
4204                .forwarding
4205                .module_id_for_connection(ctx.connection_id)
4206                .ok()
4207                .flatten()
4208                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4209            let request: ActivityRecordRequest =
4210                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4211            crate::activity::validate(&module, &request)?;
4212            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4213            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4214            let at_ms = std::time::SystemTime::now()
4215                .duration_since(std::time::UNIX_EPOCH)
4216                .unwrap_or_default()
4217                .as_millis() as u64;
4218            ledger.record(module, request, actor, at_ms).await
4219        }
4220        .await;
4221        match result {
4222            Ok(entry) => Ok(vec![control_response_body_frame(
4223                &frame,
4224                &ActivityRecordResponse { seq: entry.seq },
4225                "activity.record",
4226            )?]),
4227            Err(code) => Ok(vec![control_error_frame(
4228                &frame,
4229                code,
4230                "activity recording refused",
4231            )?]),
4232        }
4233    }
4234
4235    fn activity_actor(
4236        &self,
4237        connection: ConnectionId,
4238        module: &str,
4239        on: Option<&subc_control::ActivityOn>,
4240    ) -> Result<subc_control::ActivityActor, &'static str> {
4241        use crate::forwarding::{DataRoute, DataRouteState};
4242        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4243        let principal = |id: &str| Principal::Reserved {
4244            module_id: id.to_owned(),
4245        };
4246        let mut actor = ActivityActor {
4247            principal: principal(module),
4248            scope_owner: None,
4249            agent_id: None,
4250            run_id: None,
4251            flow_id: None,
4252            project_root: None,
4253            workdir: None,
4254            project_id: None,
4255        };
4256        let scope = match on {
4257            None => return Ok(actor),
4258            Some(ActivityOn::Scope(scope)) => {
4259                let selector = subc_protocol::scope::ScopeSelector::new(
4260                    principal(module),
4261                    scope.scope_ref.clone(),
4262                    Some(scope.epoch),
4263                );
4264                self.scopes
4265                    .read()
4266                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4267                    .attribute(&principal(module), module, &selector, true)
4268                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4269            }
4270            Some(ActivityOn::Route(route)) => {
4271                let DataRoute::Module(DataRouteState::Bound(binding)) = self
4272                    .forwarding
4273                    .lookup_data_route(connection, route.channel, route.epoch)
4274                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4275                else {
4276                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4277                };
4278                if binding.module_id != module {
4279                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4280                }
4281                actor.principal = binding.principal.clone();
4282                actor.project_root = binding
4283                    .project_root
4284                    .as_ref()
4285                    .map(|root| root.as_path().to_path_buf());
4286                let Some(bound) = &binding.scope else {
4287                    return Ok(actor);
4288                };
4289                let owner = principal(&bound.owner);
4290                let selector = subc_protocol::scope::ScopeSelector::new(
4291                    owner.clone(),
4292                    bound.scope_ref.clone(),
4293                    Some(bound.tag.scope_epoch),
4294                );
4295                let admitted = self
4296                    .scopes
4297                    .read()
4298                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4299                    .attribute(&owner, module, &selector, true)
4300                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4301                if admitted.tag != bound.tag {
4302                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4303                }
4304                admitted
4305            }
4306        };
4307        actor.scope_owner = Some(scope.stamp.owner);
4308        actor.agent_id = scope.stamp.attributes.agent_id;
4309        actor.run_id = scope.stamp.attributes.run_id;
4310        actor.flow_id = scope.stamp.attributes.flow_id;
4311        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4312        actor.project_id = scope.stamp.project_id;
4313        Ok(actor)
4314    }
4315
4316    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4317        self.refresh_capability_requirements();
4318        // A bare connection count is ambiguous between many clients holding a
4319        // route each and one client accumulating hundreds, so publish the
4320        // concentration alongside it. Route state is best-effort here: a
4321        // diagnostic endpoint must still answer if the forwarding lock is
4322        // contended.
4323        let mut counters = self.counters.snapshot();
4324        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4325            obj.insert(
4326                "activity_unparseable_lines".into(),
4327                ledger.read_errors().into(),
4328            );
4329            obj.insert(
4330                "activity_lifecycle_append_failures".into(),
4331                ledger.lifecycle_failures().into(),
4332            );
4333            obj.insert(
4334                "activity_started_digest_skips".into(),
4335                ledger.lifecycle_digest_skips().into(),
4336            );
4337        }
4338        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4339            self.forwarding.client_route_concentration(),
4340            counters.as_object_mut(),
4341        ) {
4342            obj.insert(
4343                "client_connections_with_routes".into(),
4344                connections_with_routes.into(),
4345            );
4346            obj.insert("max_routes_on_one_connection".into(), max.into());
4347        }
4348        // A module that is being fast-refused and a module that is fine look
4349        // identical from a client that retries and succeeds, so name the open
4350        // breakers here. This rides the existing free-form counters object
4351        // rather than a new wire field, so no sibling that deserializes
4352        // `ServerDescribe` has to be rebuilt to keep reading it.
4353        if let (Some(open_breakers), Some(obj)) = (
4354            self.route_bind_breakers.open_snapshot(),
4355            counters.as_object_mut(),
4356        ) {
4357            obj.insert("route_bind_breakers_open".into(), open_breakers);
4358        }
4359        // A daemon whose local endpoint could not be bound keeps serving TCP,
4360        // and the connection file names no endpoint. Name the reason here, in
4361        // the free-form counters object, so an operator reading daemon facts
4362        // sees it without searching the log, and no wire field changes.
4363        if let (Some(reason), Some(obj)) = (
4364            self.local_endpoint_unavailable.clone(),
4365            counters.as_object_mut(),
4366        ) {
4367            obj.insert("local_endpoint_unavailable".into(), reason.into());
4368        }
4369        let response = ClientControlResponse::ServerDescribe {
4370            protocol_ver: PROTOCOL_VERSION,
4371            subc_ops: subc_ops(),
4372            capabilities: self.describe_capabilities(),
4373            connected_clients: self.connected_clients.count(),
4374            counters: Some(counters),
4375            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4376            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4377            capability_requirements: self.capability_requirement_statuses(),
4378            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4379            config_path: self
4380                .rescan
4381                .as_ref()
4382                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4383        };
4384        Ok(vec![control_response_body_frame(
4385            &frame,
4386            &response,
4387            "ClientControlResponse::ServerDescribe",
4388        )?])
4389    }
4390
4391    fn handle_catalog_list(
4392        &self,
4393        frame: Frame,
4394        module_id: Option<String>,
4395    ) -> Result<Vec<Frame>, RouterError> {
4396        let (generation, modules) = self
4397            .registry
4398            .list_modules_with_nonce_verification()
4399            .map_err(|err| {
4400                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4401            })?;
4402        let entries = modules
4403            .into_iter()
4404            .filter(|(registration, _)| {
4405                module_id
4406                    .as_deref()
4407                    .map(|wanted| registration.manifest.module_id == wanted)
4408                    .unwrap_or(true)
4409            })
4410            .map(|(registration, nonce_verified)| {
4411                let not_ready = self.not_ready_reason(&registration);
4412                let roles = registration.manifest.provides;
4413                CatalogEntry::new(
4414                    registration.manifest.module_id,
4415                    roles,
4416                    registration.control_ops,
4417                )
4418                .with_ready(not_ready.is_none())
4419                .with_not_ready(not_ready)
4420                .with_module_version(Some(registration.manifest.module_version))
4421                .with_provenance(registration.manifest.provenance)
4422                .with_capabilities(registration.manifest.capabilities)
4423                .with_self_signals(registration.manifest.self_signals)
4424                .with_events(registration.manifest.events)
4425                .with_operation_delivery(registration.manifest.operation_delivery)
4426                .with_operation_payload(registration.manifest.operation_payload)
4427                .with_flow_operations(registration.manifest.flow_operations)
4428                .with_confirm_operations(registration.manifest.confirm_operations)
4429                .with_rich_ui(registration.manifest.rich_ui)
4430                .with_role_declarations(registration.manifest.role_declarations)
4431                .with_nonce_verified(nonce_verified)
4432            })
4433            .collect();
4434        let response = ClientControlResponse::CatalogList {
4435            generation,
4436            modules: entries,
4437            subc_ops: subc_ops(),
4438        };
4439        Ok(vec![control_response_body_frame(
4440            &frame,
4441            &response,
4442            "ClientControlResponse::CatalogList",
4443        )?])
4444    }
4445
4446    /// The principal a route.open's `consumer_identity` attests: `Direct`
4447    /// when it presents none, `Reserved` when its launch nonce verifies, and
4448    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4449    /// can run before any refusal check without changing which check refuses.
4450    fn attest_route_open_consumer(
4451        &self,
4452        consumer_identity: Option<&ConsumerIdentity>,
4453    ) -> Option<Principal> {
4454        let Some(consumer_identity) = consumer_identity else {
4455            return Some(Principal::Direct);
4456        };
4457        self.supervisor
4458            .spawned_consumer_authorized(
4459                &consumer_identity.module_id,
4460                &consumer_identity.launch_nonce,
4461            )
4462            .then(|| Principal::Reserved {
4463                module_id: consumer_identity.module_id.clone(),
4464            })
4465    }
4466
4467    /// The caller fields for a refusal that only has the raw request frame,
4468    /// such as the connection-level admission refusal sent before the request
4469    /// reaches [`Self::handle_route_open`].
4470    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4471        let Ok(ClientControlRequest::RouteOpen {
4472            identity,
4473            consumer_identity,
4474            scope,
4475            ..
4476        }) = parse_client_control_request(&frame.body)
4477        else {
4478            return RouteOpenCaller::unknown();
4479        };
4480        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4481        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4482    }
4483
4484    fn route_open_principal(
4485        &self,
4486        frame: &Frame,
4487        consumer_identity: Option<ConsumerIdentity>,
4488    ) -> Result<Result<Principal, Frame>, RouterError> {
4489        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4490        self.route_open_principal_attested(frame, consumer_identity, attested)
4491    }
4492
4493    /// [`Self::route_open_principal`] with `attested` already taken by
4494    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4495    /// open checks the launch nonce once.
4496    fn route_open_principal_attested(
4497        &self,
4498        frame: &Frame,
4499        consumer_identity: Option<ConsumerIdentity>,
4500        attested: Option<Principal>,
4501    ) -> Result<Result<Principal, Frame>, RouterError> {
4502        if let Some(principal) = attested {
4503            return Ok(Ok(principal));
4504        }
4505        let Some(consumer_identity) = consumer_identity else {
4506            return Ok(Ok(Principal::Direct));
4507        };
4508
4509        Ok(Err(control_error_frame(
4510            frame,
4511            "bad_consumer_identity",
4512            format!(
4513                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4514                consumer_identity.module_id
4515            ),
4516        )?))
4517    }
4518
4519    /// Ordinary `route.open` refusals go through here; admission and breaker
4520    /// refusals log separately with their capacity or breaker state. The daemon can
4521    /// attest which code it sent: without the event, a client's "the daemon
4522    /// refused me" and the daemon's own view could only be reconciled by
4523    /// argument. Malformed input (`invalid_project_root`) does not come here;
4524    /// rejecting a request that was never a valid open is not a refusal of one.
4525    #[allow(clippy::too_many_arguments)]
4526    fn route_open_refusal_frame(
4527        &self,
4528        ctx: &RouteCtx,
4529        caller: &RouteOpenCaller,
4530        frame: &Frame,
4531        module_id: &str,
4532        reason: &'static str,
4533        code: &'static str,
4534        message: impl Into<String>,
4535    ) -> Result<Frame, RouterError> {
4536        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4537        control_error_frame(frame, code, message.into())
4538    }
4539
4540    /// Answer a waiting `route.open` whose relayed bind was refused.
4541    ///
4542    /// `daemon_authored` is provenance, not a code match: the daemon
4543    /// synthesized the refusal itself (a drain, a late reply, a scope that
4544    /// ended mid-bind) or the module refused the bind with its own body. A
4545    /// daemon refusal carries the daemon-origin bit; a module's own
4546    /// rejection is relayed with the bit clear, so a module that refuses
4547    /// with `module_reloading` still cannot mint a daemon refusal.
4548    #[allow(clippy::too_many_arguments)]
4549    fn route_bind_rejection(
4550        &self,
4551        ctx: &RouteCtx,
4552        caller: &RouteOpenCaller,
4553        frame: &Frame,
4554        target_module_id: &str,
4555        body: ErrorBody,
4556        daemon_authored: bool,
4557    ) -> Result<Vec<Frame>, RouterError> {
4558        // The daemon's own commit re-check refused the bind because the
4559        // scope ended or changed after admission. The module accepted;
4560        // counting it as a module rejection would blame the module. This
4561        // branch only runs for daemon-synthesized bodies: a module that
4562        // refuses with a scope code is answered as a module rejection below.
4563        if daemon_authored {
4564            let scope_code = match body.code.as_str() {
4565                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4566                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4567                _ => None,
4568            };
4569            if let Some(code) = scope_code {
4570                self.observe_route_open_refusal(
4571                    ctx,
4572                    caller,
4573                    target_module_id,
4574                    "scope_changed_before_commit",
4575                    code,
4576                );
4577                return Ok(vec![control_error_body_frame(frame, body)?]);
4578            }
4579        }
4580        self.counters
4581            .increment_route_open_refused("module_rejected");
4582        info!(
4583            target: "control",
4584            code = "module_rejected",
4585            module_code = ?body.code,
4586            module_id = ?target_module_id,
4587            connection_id = ctx.connection_id.get(),
4588            principal = %caller.principal,
4589            scope = %caller.scope,
4590            harness = %caller.harness,
4591            "route.open refused"
4592        );
4593        let refusal = if daemon_authored {
4594            control_error_body_frame(frame, body)?
4595        } else {
4596            control_relayed_error_body_frame(frame, body)?
4597        };
4598        Ok(vec![refusal])
4599    }
4600
4601    /// Refuse a `route.open` because the target module's bind-relay breaker is
4602    /// open, without attempting the relay.
4603    ///
4604    /// The wire code is `module_timeout`, which is the truth (the module has
4605    /// not been answering binds) and which both SDKs already classify as
4606    /// retryable with capped backoff. Reusing it is what keeps this change out
4607    /// of both SDKs; the daemon-side distinction lives in the counter key
4608    /// instead.
4609    ///
4610    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4611    /// While a breaker is open this fires on every open to that module, and the
4612    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4613    /// produced 261 lines about a single module inside 3000 lines of daemon
4614    /// log. The rare transitions are logged at warn/info instead and the volume
4615    /// is carried by the counter, so the evidence survives without the flood.
4616    /// The debug line keeps a per-refusal record reachable for whoever turns
4617    /// the level up.
4618    #[allow(clippy::too_many_arguments)]
4619    fn route_open_breaker_refusal_frame(
4620        &self,
4621        ctx: &RouteCtx,
4622        caller: &RouteOpenCaller,
4623        frame: &Frame,
4624        module_id: &str,
4625        consecutive_timeouts: u32,
4626        retry_in: Duration,
4627        probe_in_flight: bool,
4628    ) -> Result<Frame, RouterError> {
4629        self.counters
4630            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4631        debug!(
4632            target: "control",
4633            code = "module_timeout",
4634            module_id = ?module_id,
4635            connection_id = ctx.connection_id.get(),
4636            principal = %caller.principal,
4637            scope = %caller.scope,
4638            harness = %caller.harness,
4639            consecutive_timeouts,
4640            retry_in_ms = retry_in.as_millis() as u64,
4641            probe_in_flight,
4642            "route.open refused by open bind-relay breaker"
4643        );
4644        // Say what a caller can act on. An open bind-relay breaker means the
4645        // module timed out accepting several new routes in a row. The module
4646        // is still running and its established routes keep working; only new
4647        // route.open requests are refused until the cooldown ends and one
4648        // test route (the probe) gets through. A message that only counts
4649        // failed relays reads as "the module is down" to a worker that sees it.
4650        let detail = if probe_in_flight {
4651            "one test route is already being tried; retry once it settles".to_string()
4652        } else {
4653            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4654        };
4655        control_error_frame(
4656            frame,
4657            "module_timeout",
4658            format!(
4659                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4660                 timed out in a row); {detail}; its established routes are unaffected"
4661            ),
4662        )
4663    }
4664
4665    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4666    /// requester's bytes (an unknown target is whatever the client sent) and
4667    /// is Debug-formatted so control characters land in the log escaped
4668    /// rather than as terminal sequences for whoever tails it.
4669    ///
4670    /// `reason` names the check that refused, because one wire code has
4671    /// several senders: after a module registers, `target_unavailable` can
4672    /// come from a missing role, an inactive registration, a supervisor that
4673    /// has not marked the process live, a missing forwarding connection, or a
4674    /// failed relay, and a log that records only the code cannot say which of
4675    /// them fired. It is a static, daemon-chosen label per branch, so it is
4676    /// safe to print plainly and stays a closed set.
4677    fn observe_route_open_refusal(
4678        &self,
4679        ctx: &RouteCtx,
4680        caller: &RouteOpenCaller,
4681        module_id: &str,
4682        reason: &'static str,
4683        code: &'static str,
4684    ) {
4685        self.counters.increment_route_open_refused(code);
4686        info!(
4687            target: "control",
4688            code,
4689            reason,
4690            module_id = ?module_id,
4691            connection_id = ctx.connection_id.get(),
4692            principal = %caller.principal,
4693            scope = %caller.scope,
4694            harness = %caller.harness,
4695            "route.open refused"
4696        );
4697        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4698            self.route_outages.record_not_serving(module_id, reason);
4699        }
4700    }
4701
4702    /// Record an ACCEPTED route.open.
4703    ///
4704    /// Refusals have been logged and counted since the attestation work; accepts
4705    /// were invisible, so the daemon knew every principal it stamped and wrote
4706    /// none of them down. The party that attests the identity was the only party
4707    /// not recording it, which left a credential vault unable to name the sender
4708    /// of a call that reached it (claustrum #43) and left the launch-nonce
4709    /// concurrency question unanswerable from the outside.
4710    ///
4711    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4712    /// `code`/`module_id`/`connection_id` returns both directions of the same
4713    /// decision rather than two shapes a reader has to join by hand.
4714    ///
4715    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4716    /// and the difference carries information rather than being an
4717    /// inconsistency. This line is only reachable after a successful bind to a
4718    /// REGISTERED module, so the value has already passed HELLO validation
4719    /// including the path-hazard refusal and cannot contain control bytes. A
4720    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4721    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4722    ///
4723    /// Bare is also what every other daemon line already emits (`module
4724    /// registered`, `configured module supervised`). Shipping `?module_id` here
4725    /// made this instrument the only one in the file whose ids did not answer
4726    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4727    /// line whose whole purpose is being grepped beside its sibling.
4728    ///
4729    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4730    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4731    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4732    /// one are recorded identically. A test written against that harness passes
4733    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4734    /// green assertion that cannot fail. The same limit applies to the escaping
4735    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4736    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4737    /// Fencing either needs the real formatter, not the capture layer.
4738    ///
4739    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4740    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4741    /// to record. The ephemeral port would decay within minutes and answer only a
4742    /// live question. The identity question is instead answered by counting
4743    /// distinct live connections presenting one module's `consumer_identity` --
4744    /// "is anyone else holding this secret" rather than "is this the right
4745    /// process".
4746    fn observe_route_open_accept(
4747        &self,
4748        ctx: &RouteCtx,
4749        module_id: &str,
4750        principal: &str,
4751        scope: &str,
4752    ) {
4753        self.route_outages.record_accepted(module_id);
4754        self.counters.increment_route_open_accepted(principal);
4755        info!(
4756            target: "control",
4757            principal,
4758            module_id,
4759            connection_id = ctx.connection_id.get(),
4760            scope = %scope,
4761            "route.open accepted"
4762        );
4763    }
4764
4765    fn supervised_absent_route_open_refusal_frame(
4766        &self,
4767        ctx: &RouteCtx,
4768        caller: &RouteOpenCaller,
4769        frame: &Frame,
4770        module_id: &str,
4771        code: &'static str,
4772        status: &crate::supervise::ModuleStatus,
4773    ) -> Result<Frame, RouterError> {
4774        self.counters.increment_route_open_refused(code);
4775        info!(
4776            target: "control",
4777            code,
4778            reason = "supervised_not_registered",
4779            module_id = ?module_id,
4780            connection_id = ctx.connection_id.get(),
4781            principal = %caller.principal,
4782            scope = %caller.scope,
4783            harness = %caller.harness,
4784            state = %status.state,
4785            enabled = status.enabled,
4786            live = status.live,
4787            "route.open refused"
4788        );
4789        // A supervised module whose process has not registered is not
4790        // serving, whatever the reason; the supervisor knows this id, so it is
4791        // safe to track.
4792        self.route_outages
4793            .record_not_serving(module_id, "supervised_not_registered");
4794        let message = if let Some(pid) = status.waiting_for_previous_pid {
4795            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4796                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4797            })
4798        } else {
4799            format!(
4800                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4801                status.state, status.enabled, status.live
4802            )
4803        };
4804        control_error_frame(frame, code, message)
4805    }
4806
4807    async fn handle_route_open(
4808        &self,
4809        ctx: &RouteCtx,
4810        frame: Frame,
4811        request: RouteOpenRequest,
4812        timings: &mut RouteOpenTimings,
4813    ) -> Result<Vec<Frame>, RouterError> {
4814        let RouteOpenRequest {
4815            target,
4816            mut identity,
4817            consumer_identity,
4818            consumer_capabilities,
4819            role_versions,
4820            busy_refusals,
4821            admission_facts,
4822            scope,
4823        } = request;
4824        let target_module_id = target_module_id(&target).to_string();
4825        timings.module_id.clone_from(&target_module_id);
4826        if self
4827            .registry
4828            .get_module_by_connection(ctx.connection_id)
4829            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4830            .is_some()
4831        {
4832            return Ok(vec![control_error_frame(
4833                &frame,
4834                "invalid_request",
4835                "module connections cannot open client routes",
4836            )?]);
4837        }
4838        debug!(
4839            connection_id = ctx.connection_id.get(),
4840            corr = frame.header.corr,
4841            module_id = %target_module_id,
4842            "handling route.open"
4843        );
4844
4845        // The caller's attested principal, requested scope and harness, for
4846        // every `route.open refused` line below. The launch nonce is checked
4847        // here, once, and the answer is reused where the principal is
4848        // enforced further down, so the order of refusals is unchanged.
4849        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4850        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4851
4852        // A malformed declaration is refused first, before anything about the
4853        // target is looked up: the same body would be refused against any
4854        // module, so the caller learns nothing by retrying or waiting. An empty
4855        // map declares nothing and travels as no field at all, so a provider
4856        // only ever sees a missing field or a non-empty one.
4857        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4858        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4859            self.observe_route_open_refusal(
4860                ctx,
4861                &caller,
4862                &target_module_id,
4863                "invalid_role_versions",
4864                error_codes::INVALID_REQUEST,
4865            );
4866            return Ok(vec![control_error_body_frame(
4867                &frame,
4868                ErrorBody {
4869                    code: error_codes::INVALID_REQUEST.to_string(),
4870                    message: error.to_string(),
4871                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4872                },
4873            )?]);
4874        }
4875
4876        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4877            if status.waiting_for_previous_pid.is_some() {
4878                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4879                    ctx,
4880                    &caller,
4881                    &frame,
4882                    &target_module_id,
4883                    "module_warming",
4884                    &status,
4885                )?]);
4886            }
4887        }
4888
4889        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4890        // opposite. Below, a caller learns whether a module is unregistered,
4891        // supervised-but-down (with state/enabled/live), or registered without the
4892        // requested role. Elsewhere that is an enumeration leak: a probe learning
4893        // the shape of a fleet it cannot otherwise see.
4894        //
4895        // It is not one here, and the reason is the ACCESS MODEL rather than
4896        // anything about these errors. Reaching route.open requires the
4897        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4898        // connection file, so any caller who completes it already runs as this
4899        // user -- and can read subc.jsonc for the module list and `ck module
4900        // status` for live state. The reply discloses nothing the caller cannot
4901        // read more easily from disk, while the precision is load-bearing:
4902        // `unknown_module` is retryable and a missing role is not.
4903        //
4904        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4905        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4906        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4907        let Some(registration) = self
4908            .registry
4909            .get_module(&target_module_id)
4910            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4911        else {
4912            if let Some((status, warming)) =
4913                self.supervisor_status(&target_module_id, frame.header.corr)?
4914            {
4915                // BEFORE the two availability codes below, because for a module
4916                // that speaks no subc wire both of them are false comfort: they
4917                // say "not right now" and are retried, and this module will
4918                // never register no matter how long the caller waits. The
4919                // absence here is the declaration being honoured, not a module
4920                // that is late.
4921                if status.protocol == ModuleProtocol::None {
4922                    return Ok(vec![self.route_open_refusal_frame(
4923                        ctx,
4924                        &caller,
4925                        &frame,
4926                        &target_module_id,
4927                        "protocol_none",
4928                        error_codes::MODULE_NO_PROTOCOL,
4929                        format!(
4930                            "module_id '{target_module_id}' is declared protocol: none; \
4931                             it speaks no subc wire and serves no routes"
4932                        ),
4933                    )?]);
4934                }
4935                let code = if warming {
4936                    "module_warming"
4937                } else {
4938                    "target_unavailable"
4939                };
4940                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4941                    ctx,
4942                    &caller,
4943                    &frame,
4944                    &target_module_id,
4945                    code,
4946                    &status,
4947                )?]);
4948            }
4949            if let Some(removed_ago_ms) =
4950                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4951            {
4952                return Ok(vec![self.route_open_refusal_frame(
4953                    ctx,
4954                    &caller,
4955                    &frame,
4956                    &target_module_id,
4957                    "removed",
4958                    error_codes::MODULE_REMOVED,
4959                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4960                )?]);
4961            }
4962            return Ok(vec![self.route_open_refusal_frame(
4963                ctx,
4964                &caller,
4965                &frame,
4966                &target_module_id,
4967                "not_registered",
4968                error_codes::UNKNOWN_MODULE,
4969                format!("module_id '{target_module_id}' is not registered"),
4970            )?]);
4971        };
4972
4973        // Best-effort only: registry readiness and forwarding reservation use
4974        // different locks, so a module can flip readiness between this read and
4975        // the relay. Modules must still tolerate an `on_bind` while not ready.
4976        if !registration.ready {
4977            self.counters
4978                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4979            info!(
4980                target: "control",
4981                code = error_codes::MODULE_WARMING,
4982                module_id = ?target_module_id,
4983                connection_id = ctx.connection_id.get(),
4984                principal = %caller.principal,
4985                scope = %caller.scope,
4986                harness = %caller.harness,
4987                reason = "declared_not_ready",
4988                "route.open refused"
4989            );
4990            // The module is registered but says it cannot take work, which is
4991            // an outage from the caller's side even though its process is up.
4992            self.route_outages
4993                .record_not_serving(&target_module_id, "declared_not_ready");
4994            return Ok(vec![control_error_body_frame(
4995                &frame,
4996                ErrorBody {
4997                    code: error_codes::MODULE_WARMING.to_string(),
4998                    message: format!(
4999                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5000                    ),
5001                    detail: Some(serde_json::json!({
5002                        "reason": "declared_not_ready"
5003                    })),
5004                },
5005            )?]);
5006        }
5007
5008        // Effective readiness, second half: a module that declares a capability
5009        // `need: required` is not routable while that capability has no
5010        // registered provider. It is enforced HERE, as a retryable routing
5011        // refusal, and deliberately not as spawn ordering or a boot block. The
5012        // module is still started and registered and can make its own calls;
5013        // spawn ordering is a promise that cannot be kept once a provider
5014        // crashes at runtime, and refusing to boot would stop the whole
5015        // machine, including the tools needed to fix its configuration.
5016        //
5017        // "Provided" is the evaluator's verdict, which counts a provider as
5018        // soon as it has REGISTERED, not once it is ready. Two modules that
5019        // require each other's capabilities are therefore both routable once
5020        // both register; counting readiness instead would deadlock them.
5021        //
5022        // Only new opens are refused. Routes already bound when a provider
5023        // goes away stay bound: nothing here tears them down, and the module
5024        // answers them as it can. Like the readiness read above this is
5025        // best-effort against a provider registering or leaving concurrently.
5026        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
5027            self.counters
5028                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5029            info!(
5030                target: "control",
5031                code = error_codes::MODULE_WARMING,
5032                module_id = ?target_module_id,
5033                connection_id = ctx.connection_id.get(),
5034                principal = %caller.principal,
5035                scope = %caller.scope,
5036                harness = %caller.harness,
5037                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5038                capability = %capability,
5039                "route.open refused"
5040            );
5041            return Ok(vec![control_error_body_frame(
5042                &frame,
5043                ErrorBody {
5044                    code: error_codes::MODULE_WARMING.to_string(),
5045                    message: format!(
5046                        "module_id '{target_module_id}' requires capability '{capability}', \
5047                         which no registered module provides; retry"
5048                    ),
5049                    detail: Some(serde_json::json!({
5050                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5051                        "capability": capability,
5052                    })),
5053                },
5054            )?]);
5055        }
5056
5057        if !target_has_required_role(&target, &registration.manifest.provides) {
5058            return Ok(vec![self.route_open_refusal_frame(
5059                ctx,
5060                &caller,
5061                &frame,
5062                &target_module_id,
5063                "role_not_provided",
5064                "target_unavailable",
5065                format!("module_id '{target_module_id}' does not provide the requested target"),
5066            )?]);
5067        }
5068
5069        if registration.state != ChannelState::Active {
5070            return Ok(vec![self.route_open_refusal_frame(
5071                ctx,
5072                &caller,
5073                &frame,
5074                &target_module_id,
5075                "registration_not_active",
5076                "target_unavailable",
5077                format!("module_id '{target_module_id}' is not active"),
5078            )?]);
5079        }
5080
5081        if self
5082            .forwarding
5083            .module_is_draining(&target_module_id)
5084            .map_err(RouterError::Forwarding)?
5085        {
5086            return Ok(vec![self.route_open_refusal_frame(
5087                ctx,
5088                &caller,
5089                &frame,
5090                &target_module_id,
5091                "reloading",
5092                "module_reloading",
5093                format!("module_id '{target_module_id}' is reloading"),
5094            )?]);
5095        }
5096
5097        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5098            process_liveness.process_live(&target_module_id) == Some(false)
5099        }) {
5100            // A module the supervisor is restarting or reloading can still hold
5101            // a registration: the old process before its connection closes, or
5102            // a new one that registered while the supervisor was draining. The
5103            // forwarding table does not see that as draining, but the consumer
5104            // should still be told to retry soon, exactly as for the drain
5105            // above, rather than that the target is unavailable.
5106            if process_liveness.process_replacing(&target_module_id) {
5107                return Ok(vec![self.route_open_refusal_frame(
5108                    ctx,
5109                    &caller,
5110                    &frame,
5111                    &target_module_id,
5112                    "reloading",
5113                    "module_reloading",
5114                    format!("module_id '{target_module_id}' is reloading"),
5115                )?]);
5116            }
5117            return Ok(vec![self.route_open_refusal_frame(
5118                ctx,
5119                &caller,
5120                &frame,
5121                &target_module_id,
5122                "supervisor_not_live",
5123                "target_unavailable",
5124                format!("module_id '{target_module_id}' is not live"),
5125            )?]);
5126        }
5127
5128        if !self
5129            .forwarding
5130            .has_live_module_connection(&target_module_id)
5131            .map_err(RouterError::Forwarding)?
5132        {
5133            return Ok(vec![self.route_open_refusal_frame(
5134                ctx,
5135                &caller,
5136                &frame,
5137                &target_module_id,
5138                "no_forwarding_connection",
5139                "target_unavailable",
5140                format!("module_id '{target_module_id}' has no live forwarding connection"),
5141            )?]);
5142        }
5143
5144        if let Some(error) =
5145            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5146        {
5147            self.observe_route_open_refusal(
5148                ctx,
5149                &caller,
5150                &target_module_id,
5151                "op_not_allowed",
5152                "op_not_allowed",
5153            );
5154            return Ok(vec![error]);
5155        }
5156
5157        let principal =
5158            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5159                Ok(principal) => principal,
5160                Err(error) => {
5161                    self.observe_route_open_refusal(
5162                        ctx,
5163                        &caller,
5164                        &target_module_id,
5165                        "bad_consumer_identity",
5166                        "bad_consumer_identity",
5167                    );
5168                    return Ok(vec![error]);
5169                }
5170            };
5171        timings.principal = principal_label(&principal);
5172
5173        // Only Callosum can attest a production phone build. Consume the claim
5174        // here so it can be kept on the route without reaching any module.
5175        let build_class = identity.build_class.take().filter(|build_class| {
5176            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5177                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5178            } else {
5179                if self
5180                    .build_class_warned
5181                    .lock()
5182                    .unwrap_or_else(|p| p.into_inner())
5183                    .insert(ctx.connection_id)
5184                {
5185                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5186                        "ignoring build_class from a non-Callosum opener");
5187                }
5188                false
5189            }
5190        });
5191
5192        // This is attested, control-plane policy for supervised module origins.
5193        // Keep it before route reservation and out of the opaque forwarding hot
5194        // path: data frames must never acquire a per-frame capability check.
5195        if let Principal::Reserved {
5196            module_id: opening_module_id,
5197        } = &principal
5198        {
5199            if let Some(opening_registration) = self
5200                .registry
5201                .get_module(opening_module_id)
5202                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5203            {
5204                if let Some(capability) =
5205                    denied_capability(&opening_registration.manifest, &registration.manifest)
5206                {
5207                    warn!(
5208                        opening_module_id,
5209                        target_module_id,
5210                        capability,
5211                        "refusing route.open because an attested capability deny edge matches"
5212                    );
5213                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5214                        &frame,
5215                        &target_module_id,
5216                        "capability_deny_edge",
5217                        "capability_forbidden",
5218                        format!(
5219                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5220                        ),
5221                    )?]);
5222                }
5223            }
5224        }
5225
5226        if admission_facts.is_some() {
5227            let carrier_matches = matches!(
5228                &principal,
5229                Principal::Reserved { module_id }
5230                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5231            );
5232            if !carrier_matches {
5233                return Ok(vec![self.route_open_refusal_frame(
5234                    ctx,
5235                    &caller,
5236                    &frame,
5237                    &target_module_id,
5238                    "admission_facts_carrier_not_permitted",
5239                    "admission_facts_not_permitted",
5240                    "admission facts may only be carried by the configured reserved module",
5241                )?]);
5242            }
5243
5244            let target_allowed = self
5245                .admission_facts_targets
5246                .as_ref()
5247                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5248            if !target_allowed {
5249                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5250                    &frame,
5251                    &target_module_id,
5252                    "admission_facts_target_not_listed",
5253                    "admission_facts_target_not_allowed",
5254                    format!(
5255                        "admission facts are not permitted for target module_id '{target_module_id}'"
5256                    ),
5257                )?]);
5258            }
5259
5260            // Keep the value opaque to subc. The downstream admission validator owns
5261            // schema and semantic checks; this daemon only enforces carrier authority
5262            // and the configured destination allowlist.
5263        }
5264
5265        // Scope admission, on the attested principal above and never on the
5266        // request body. The tag read here travels with the pending bind and is
5267        // compared with the published one at commit, so a sync between here
5268        // and the module's ack refuses the open instead of binding a stamp
5269        // that is no longer true.
5270        let (bound_scope, mut scope_stamp) = match &scope {
5271            None => (None, None),
5272            Some(selector) => {
5273                let owner_configured = match &selector.owner {
5274                    // A reserved owner counts as configured from before its
5275                    // process is spawned (see `SupervisorHandle::is_configured`).
5276                    // So an owner that has not synced its scopes yet is refused
5277                    // as retryable (`scope_not_synced`), not as one that will
5278                    // never sync.
5279                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5280                    _ => false,
5281                };
5282                let mut scopes = self
5283                    .scopes
5284                    .write()
5285                    .unwrap_or_else(|poisoned| poisoned.into_inner());
5286                let admitted = scopes
5287                    .admit(&principal, &target_module_id, selector, owner_configured)
5288                    .and_then(|admission| {
5289                        crate::scopes::check_target_flow_support(
5290                            &admission.stamp,
5291                            &target_module_id,
5292                            registration.manifest.capabilities.as_ref(),
5293                        )?;
5294                        crate::scopes::check_target_agent_run_support(
5295                            &admission.stamp,
5296                            &target_module_id,
5297                            registration.manifest.capabilities.as_ref(),
5298                        )?;
5299                        let bound = scopes.reserve_bind(
5300                            &admission,
5301                            &principal,
5302                            ctx.connection_id,
5303                            selector.hold_presence,
5304                        );
5305                        Ok((admission, bound))
5306                    });
5307                match admitted {
5308                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5309                    Err(refusal) => {
5310                        return Ok(vec![self.route_open_refusal_frame(
5311                            ctx,
5312                            &caller,
5313                            &frame,
5314                            &target_module_id,
5315                            refusal.code,
5316                            refusal.code,
5317                            refusal.message,
5318                        )?]);
5319                    }
5320                }
5321            }
5322        };
5323        // From here on a refusal names the scope by the stamp admission
5324        // issued, exactly as the `route.open accepted` line does.
5325        if let Some(stamp) = &scope_stamp {
5326            caller.scope = route_open_scope_label(stamp);
5327        }
5328
5329        // Allow resolvable missing-tail paths so a vanished directory does not
5330        // prevent reopening an existing session for cleanup. Binding does not
5331        // authorize creation of durable provider state.
5332        //
5333        // Resolve real paths rather than lexical cleanup: canonicalize the
5334        // longest existing ancestor and append the missing tail, retaining the
5335        // path's real existing ancestry even when a leaf is missing.
5336        timings.enter(RouteOpenPhase::ProjectRoot);
5337        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5338            Ok(project) => project,
5339            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5340        };
5341        timings.enter(RouteOpenPhase::Admission);
5342        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5343            Ok(root) => root,
5344            Err(err) => {
5345                return Ok(vec![control_error_frame(
5346                    &frame,
5347                    "invalid_project_root",
5348                    err.to_string(),
5349                )?])
5350            }
5351        };
5352        // Compute compatibility project fields before stripping optional stamp
5353        // members. Preserve the canonical opener root and its project ID,
5354        // including absence. For rootless openers, use stamp workdir or `/`, and
5355        // opener ID, else stamp ID, else no ID. Emit these identity fields only
5356        // to targets without bind-no-project/v1.
5357        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5358
5359        // Last gate before any relay work, and deliberately after the cheap
5360        // registry and availability checks above: those name a more precise
5361        // condition (unknown, removed, reloading) and a caller is better served
5362        // by the precise code than by this one.
5363        //
5364        // Everything below this point costs an egress permit, a reserved handle
5365        // pair and, if the module does not answer, the whole relay budget. The
5366        // reader no longer waits for that budget, so cap each target explicitly;
5367        // serial dispatch used to provide the accidental cap of one relay per
5368        // connection. Admission is a mutex-protected count and never waits.
5369        let _concurrency_guard = match self
5370            .route_bind_concurrency
5371            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5372        {
5373            Ok(guard) => guard,
5374            Err(in_flight) => {
5375                return Ok(vec![self.route_open_target_capacity_refusal(
5376                    ctx,
5377                    &caller,
5378                    &frame,
5379                    &target_module_id,
5380                    in_flight,
5381                )?]);
5382            }
5383        };
5384
5385        // A module that has already burned the whole budget `threshold` times
5386        // in a row does not get to charge it again until a probe says it recovered.
5387        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5388            RouteBindAdmission::Admitted { guard, probe } => {
5389                if probe {
5390                    info!(
5391                        module_id = %target_module_id,
5392                        connection_id = ctx.connection_id.get(),
5393                        "route.bind breaker half-open: admitting one probe"
5394                    );
5395                }
5396                guard
5397            }
5398            RouteBindAdmission::Refused {
5399                consecutive_timeouts,
5400                retry_in,
5401                probe_in_flight,
5402            } => {
5403                return Ok(vec![self.route_open_breaker_refusal_frame(
5404                    ctx,
5405                    &caller,
5406                    &frame,
5407                    &target_module_id,
5408                    consecutive_timeouts,
5409                    retry_in,
5410                    probe_in_flight,
5411                )?]);
5412            }
5413        };
5414
5415        // Resolve the per-module budget here so the wait matches the operator's
5416        // intent for this specific target. A per-module override in
5417        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5418        // daemons) wins over the daemon-wide default.
5419        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5420        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5421        let operator_app = self
5422            .operator_apps
5423            .lock()
5424            .unwrap_or_else(|p| p.into_inner())
5425            .get(&ctx.connection_id)
5426            .cloned();
5427        timings.enter(RouteOpenPhase::EgressReservation);
5428        let pending = match self
5429            .forwarding
5430            .begin_route_bind_relay_for(
5431                ctx.connection_id,
5432                ctx.egress.clone(),
5433                response_version(&frame),
5434                frame.header.corr,
5435                &target_module_id,
5436                principal.clone(),
5437                bound_scope,
5438                project_root,
5439                identity.harness.clone(),
5440                build_class,
5441                operator_app.clone(),
5442                relay_deadline,
5443                busy_refusals == Some(true),
5444            )
5445            .await
5446        {
5447            Ok(pending) => pending,
5448            Err(err) => {
5449                return Ok(vec![self.route_open_refusal_frame(
5450                    ctx,
5451                    &caller,
5452                    &frame,
5453                    &target_module_id,
5454                    "relay_reservation_failed",
5455                    forwarding_error_code(&err),
5456                    err.to_string(),
5457                )?])
5458            }
5459        };
5460        timings.enter(RouteOpenPhase::Admission);
5461        let crate::forwarding::PendingRouteBindRelay {
5462            endpoint,
5463            module_sink,
5464            negotiated_ver,
5465            client_channel,
5466            client_epoch,
5467            module_channel,
5468            module_epoch,
5469            corr: relay_corr,
5470            receiver,
5471        } = pending;
5472        let mut reservation =
5473            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5474
5475        // Reserving egress can wait while a module reconnects or a swap cuts
5476        // over. Check the relay's captured connection, not the earlier by-id
5477        // lookup: the original module's flow/run capabilities cannot authorize
5478        // its replacement. The captured sink stays bound to that connection.
5479        let relay_registration = self
5480            .registry
5481            .get_module_by_connection(endpoint.connection_id)
5482            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5483        let relay_capabilities = relay_registration
5484            .as_ref()
5485            .and_then(|registration| registration.manifest.capabilities.as_ref());
5486        if let Some(selector) = &scope {
5487            // Egress reservation may wait across a scope update. Recheck the
5488            // proof against the current record before sending the captured bind.
5489            if let Err(refusal) = self
5490                .scopes
5491                .read()
5492                .unwrap_or_else(|poisoned| poisoned.into_inner())
5493                .check_bind_proof(&principal, selector)
5494            {
5495                reservation.release_and_disarm();
5496                return Ok(vec![self.route_open_refusal_frame(
5497                    ctx,
5498                    &caller,
5499                    &frame,
5500                    &target_module_id,
5501                    refusal.code,
5502                    refusal.code,
5503                    refusal.message,
5504                )?]);
5505            }
5506        }
5507        if let Some(stamp) = scope_stamp
5508            .as_ref()
5509            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5510        {
5511            if let Err(refusal) = crate::scopes::check_target_flow_support(
5512                stamp,
5513                &target_module_id,
5514                relay_capabilities,
5515            )
5516            .and_then(|()| {
5517                crate::scopes::check_target_agent_run_support(
5518                    stamp,
5519                    &target_module_id,
5520                    relay_capabilities,
5521                )
5522            }) {
5523                reservation.release_and_disarm();
5524                return Ok(vec![self.route_open_refusal_frame(
5525                    ctx,
5526                    &caller,
5527                    &frame,
5528                    &target_module_id,
5529                    refusal.code,
5530                    refusal.code,
5531                    refusal.message,
5532                )?]);
5533            }
5534        }
5535
5536        // Older readers may reject unknown fields. Select optional fields from
5537        // the captured endpoint's declaration, including on unscoped binds.
5538        if !relay_capabilities.is_some_and(|caps| {
5539            caps.provides
5540                .iter()
5541                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5542        }) {
5543            if let Some(stamp) = scope_stamp.as_mut() {
5544                stamp.workdir = None;
5545                stamp.project_id = None;
5546            }
5547        }
5548        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5549        // attribution. The run gate above has already required the target to
5550        // declare `worker-runs/v1`, which promises to read both
5551        // `on_behalf_of_agent` and `call_mode`.
5552        let worker_run = scope_stamp
5553            .as_ref()
5554            .is_some_and(crate::scopes::is_worker_run)
5555            && relay_capabilities.is_some_and(|caps| {
5556                caps.provides
5557                    .iter()
5558                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5559            });
5560        if !worker_run
5561            && !relay_capabilities.is_some_and(|caps| {
5562                caps.provides
5563                    .iter()
5564                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5565            })
5566        {
5567            if let Some(stamp) = scope_stamp.as_mut() {
5568                stamp.attributes.on_behalf_of_agent = None;
5569            }
5570        }
5571        if !relay_capabilities.is_some_and(|caps| {
5572            caps.provides
5573                .iter()
5574                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5575        }) {
5576            if let Some(stamp) = scope_stamp.as_mut() {
5577                stamp.attributes.flow_version = None;
5578                stamp.attributes.flow_code_hash = None;
5579                stamp.attributes.flow_install_id = None;
5580            }
5581        }
5582        let call_mode = if worker_run
5583            || relay_capabilities
5584                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5585        {
5586            Some(derive_call_mode(scope_stamp.as_ref()))
5587        } else {
5588            if let Some(stamp) = scope_stamp.as_mut() {
5589                stamp.expires_at_ms = None;
5590            }
5591            None
5592        };
5593
5594        debug!(
5595            connection_id = ctx.connection_id.get(),
5596            client_channel,
5597            client_epoch,
5598            module_channel,
5599            module_epoch,
5600            "reserved route handle pair"
5601        );
5602        // Rendered BEFORE the move into the relay, because the accept arm below
5603        // is where it is logged and the principal is gone by then.
5604        let route_principal_label = principal_label(&principal);
5605        let relay = ModuleControlRequest::RouteBind {
5606            route_channel: module_channel,
5607            epoch: module_epoch,
5608            target,
5609            identity,
5610            principal: Some(principal),
5611            consumer_capabilities,
5612            role_versions,
5613            admission_facts,
5614            scope: scope_stamp,
5615            call_mode,
5616        };
5617        let peer = RouteBindPeer::new()
5618            .with_observed_peer(
5619                relay_capabilities
5620                    .is_some_and(|caps| {
5621                        caps.provides
5622                            .iter()
5623                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5624                    })
5625                    .then(|| {
5626                        self.observed_peers
5627                            .lock()
5628                            .unwrap_or_else(|p| p.into_inner())
5629                            .get(&ctx.connection_id)
5630                            .cloned()
5631                            .unwrap_or_else(ObservedPeer::unobserved)
5632                    }),
5633            )
5634            .with_operator_app(
5635                self.connection_operator_app(operator_app.as_ref())
5636                    .filter(|_| {
5637                        relay_capabilities.is_some_and(|caps| {
5638                            caps.provides
5639                                .iter()
5640                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5641                        })
5642                    }),
5643            );
5644        let relay_body = crate::bind_compat::encode_bind(
5645            &relay,
5646            &peer,
5647            &legacy_bind_project,
5648            relay_capabilities,
5649        )
5650        .map_err(|err| {
5651            RouterError::backend(
5652                0,
5653                frame.header.corr,
5654                format!("failed to encode route.bind request: {err}"),
5655            )
5656        })?;
5657        let relay_frame = Frame::build_with_version(
5658            negotiated_ver,
5659            FrameType::Request,
5660            control_flags(),
5661            0,
5662            0,
5663            relay_corr,
5664            relay_body,
5665        )
5666        .map_err(RouterError::FrameBuild)?;
5667
5668        timings.enter(RouteOpenPhase::BindSend);
5669        let sent = module_sink.send(relay_frame).await;
5670        timings.enter(RouteOpenPhase::Admission);
5671        if let Err(err) = sent {
5672            reservation.release_and_disarm();
5673            return Ok(vec![self.route_open_refusal_frame(
5674                ctx,
5675                &caller,
5676                &frame,
5677                &target_module_id,
5678                "relay_send_failed",
5679                "target_unavailable",
5680                err.to_string(),
5681            )?]);
5682        }
5683
5684        #[cfg(test)]
5685        if let Some(hold) = &self.route_bind_relay_sent_hold {
5686            // First wait: tell the test the bind is queued. Second wait: stay
5687            // here until the test has driven whatever it wants to happen
5688            // between queueing the bind and marking it enqueued.
5689            hold.wait().await;
5690            hold.wait().await;
5691        }
5692
5693        // A missing relay is not by itself an abandoned bind: the module may
5694        // already have acked it and the route may be committed. Only a relay
5695        // dropped without any module reply needs this GOODBYE; one a reply
5696        // settled has already had its GOODBYE decided by the reply path.
5697        if self
5698            .forwarding
5699            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5700            .map_err(RouterError::Forwarding)?
5701            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5702        {
5703            self.send_abandoned_route_bind_goodbye(
5704                &module_sink,
5705                negotiated_ver,
5706                module_channel,
5707                module_epoch,
5708            );
5709        }
5710
5711        timings.enter(RouteOpenPhase::BindAckWait);
5712        let outcome = timeout_at(relay_deadline, receiver).await;
5713        timings.enter(RouteOpenPhase::Admission);
5714        match outcome {
5715            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5716                reservation.disarm();
5717                if breaker.record_accepted() {
5718                    info!(
5719                        module_id = %target_module_id,
5720                        "route.bind breaker closed: the probe was accepted"
5721                    );
5722                }
5723                self.observe_route_open_accept(
5724                    ctx,
5725                    &target_module_id,
5726                    &route_principal_label,
5727                    &caller.scope,
5728                );
5729                Ok(Vec::new())
5730            }
5731            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5732                reservation.release_and_disarm();
5733                // A module that says no in microseconds is healthy. Rejection
5734                // is a different condition with its own refusal and must not
5735                // move the breaker.
5736                breaker.record_inconclusive();
5737                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5738            }
5739            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5740                reservation.release_and_disarm();
5741                breaker.record_inconclusive();
5742                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5743            }
5744            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5745                reservation.release_and_disarm();
5746                breaker.record_inconclusive();
5747                // Fires when the module's connection closes while a relayed
5748                // bind is pending -- typically a caller racing a module restart
5749                // whose bind was relayed BEFORE the drain mark went up. Logged
5750                // because the caller sees only its own error and the fleet has
5751                // already spent one diagnosis round unable to tell this arm
5752                // from a relay timeout without daemon-side evidence.
5753                tracing::warn!(
5754                    module_id = %target_module_id,
5755                    "route.bind relay abandoned: {message}"
5756                );
5757                Ok(vec![self.route_open_refusal_frame(
5758                    ctx,
5759                    &caller,
5760                    &frame,
5761                    &target_module_id,
5762                    "relay_abandoned",
5763                    "target_unavailable",
5764                    message,
5765                )?])
5766            }
5767            Ok(Err(_)) => {
5768                reservation.release_and_disarm();
5769                breaker.record_inconclusive();
5770                Ok(vec![self.route_open_refusal_frame(
5771                    ctx,
5772                    &caller,
5773                    &frame,
5774                    &target_module_id,
5775                    "relay_waiter_canceled",
5776                    "target_unavailable",
5777                    "route.bind relay waiter was canceled before the module responded",
5778                )?])
5779            }
5780            Err(_) => {
5781                reservation.release_and_disarm();
5782                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5783                // answer at all is the one condition a fast refusal can
5784                // usefully stand in for; every other arm already answered.
5785                if let Some(opened) = breaker.record_timeout(
5786                    self.route_bind_breaker_threshold,
5787                    self.route_bind_breaker_cooldown,
5788                ) {
5789                    warn!(
5790                        module_id = %target_module_id,
5791                        consecutive_timeouts = opened.consecutive_timeouts,
5792                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5793                        reopened_after_probe = opened.reopened_after_probe,
5794                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5795                    );
5796                }
5797                // The generous budget just burned to no answer: the module is
5798                // registered and its connection is up, but its bind handler sat
5799                // on the ack for the full budget (warm-on-bind, cold configure,
5800                // or a wedged handler). Every earlier unavailability shape
5801                // fast-refuses BEFORE the relay, so this arm firing means the
5802                // slowness is module-side -- log it so the per-module timeline
5803                // is reconstructable without client audit rows.
5804                tracing::warn!(
5805                    module_id = %target_module_id,
5806                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5807                    "route.bind relay timed out: module did not ack within budget"
5808                );
5809                Ok(vec![self.route_open_refusal_frame(
5810                    ctx,
5811                    &caller,
5812                    &frame,
5813                    &target_module_id,
5814                    "relay_timed_out",
5815                    "module_timeout",
5816                    format!(
5817                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5818                        route_bind_relay_timeout
5819                    ),
5820                )?])
5821            }
5822        }
5823    }
5824
5825    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5826        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5827            snapshot: self.supervisor.spawn_snapshot(),
5828        };
5829        Ok(vec![control_response_body_frame(
5830            &frame,
5831            &response,
5832            "ClientControlResponse::SupervisorSpawnSnapshot",
5833        )?])
5834    }
5835
5836    fn handle_supervisor_spawn_subscribe(
5837        &self,
5838        ctx: &RouteCtx,
5839        frame: Frame,
5840        since: Option<SpawnCursor>,
5841    ) -> Result<Vec<Frame>, RouterError> {
5842        match self.supervisor.subscribe_spawns(
5843            ctx.connection_id,
5844            frame.header.corr,
5845            response_version(&frame),
5846            since,
5847            ctx.egress.clone(),
5848        ) {
5849            Ok(()) => Ok(Vec::new()),
5850            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5851                Ok(vec![control_error_body_frame(
5852                    &frame,
5853                    ErrorBody {
5854                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5855                        message: "spawn cursor belongs to a different daemon incarnation"
5856                            .to_string(),
5857                        detail: Some(serde_json::json!({
5858                            "current_daemon_incarnation": current
5859                        })),
5860                    },
5861                )?])
5862            }
5863            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5864                &frame,
5865                ErrorBody {
5866                    code: "spawn_cursor_too_old".to_string(),
5867                    message: "spawn cursor predates the retained event ring".to_string(),
5868                    detail: Some(serde_json::json!({
5869                        "oldest_retained_cursor": oldest
5870                    })),
5871                },
5872            )?]),
5873            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5874                0,
5875                frame.header.corr,
5876                format!("failed to open supervisor spawn subscription: {error}"),
5877            )),
5878        }
5879    }
5880
5881    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882        let generation = self
5883            .registry
5884            .generation()
5885            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5886        let supervised = self.supervisor.list();
5887        let targets = supervised
5888            .iter()
5889            .map(|module| module.subtree_target())
5890            .collect();
5891        let subtree = tokio::spawn(crate::subtree_resources::read(targets));
5892        let mut modules = Vec::new();
5893        for module in supervised {
5894            let status = module.status_for_control("list").map_err(|err| {
5895                RouterError::backend(
5896                    0,
5897                    frame.header.corr,
5898                    format!("failed to read supervisor status: {err}"),
5899                )
5900            })?;
5901            let (configured, _) = module.configuration().map_err(|err| {
5902                RouterError::backend(
5903                    0,
5904                    frame.header.corr,
5905                    format!("failed to read module configuration: {err}"),
5906                )
5907            })?;
5908            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5909                RouterError::backend(
5910                    0,
5911                    frame.header.corr,
5912                    format!("failed to read retry time: {err}"),
5913                )
5914            })?;
5915            // Status and configuration snapshots release their locks before the image probe awaits.
5916            let image = module.running_image_agreement().await;
5917            // Read per request so the figure is current when the operator asks;
5918            // the daemon samples nothing in between.
5919            let resources = Some(crate::subtree_resources::read_process(module.clone()).await);
5920            let mut verdict =
5921                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5922            verdict.spec = Some(launch_spec_agreement(
5923                &configured,
5924                status.process_alive,
5925                status.launch_spec_digest.as_ref(),
5926            ));
5927            let pending_reload = Some(verdict);
5928            let configured_enabled = self
5929                .capability_evaluator
5930                .configured_enabled(&status.module_id);
5931            // Keep the retired policy field on the wire for one release so
5932            // existing status consumers still receive the platform policy.
5933            modules.push(
5934                SupervisorEntry::new(
5935                    status.module_id,
5936                    status.state.to_string(),
5937                    status.enabled,
5938                    status.live,
5939                    status.health.status,
5940                )
5941                .with_configured_enabled(configured_enabled)
5942                .with_next_retry_at_ms(next_retry_at_ms)
5943                .with_launch_nonce_env(Some(!cfg!(unix)))
5944                .with_launch_nonce_source(status.launch_nonce_source)
5945                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5946                .with_protocol(status.protocol)
5947                .with_pending_reload(pending_reload)
5948                .with_last_probe_ms(status.health.last_probe_ms)
5949                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5950                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5951                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5952                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5953                .with_restart_count(Some(status.restart_count))
5954                .with_max_restarts(Some(status.max_restarts))
5955                .with_lifetime_restarts(Some(status.lifetime_restarts))
5956                .with_spawn_generation(Some(status.spawn_generation))
5957                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5958                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5959                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5960                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5961                .with_resources(resources),
5962            );
5963        }
5964        let readings = subtree
5965            .await
5966            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5967        for (module, reading) in modules.iter_mut().zip(readings) {
5968            if reading.reason.as_deref() == Some("timeout") {
5969                crate::subtree_resources::log_timeout(&module.module_id, "subtree");
5970            }
5971            module.subtree = Some(reading);
5972        }
5973        let response = ClientControlResponse::SupervisorList {
5974            generation,
5975            modules,
5976        };
5977        Ok(vec![control_response_body_frame(
5978            &frame,
5979            &response,
5980            "ClientControlResponse::SupervisorList",
5981        )?])
5982    }
5983
5984    fn handle_supervisor_stderr_tail(
5985        &self,
5986        frame: Frame,
5987        module_id: String,
5988        max_lines: Option<u32>,
5989        max_bytes: Option<u32>,
5990    ) -> Result<Vec<Frame>, RouterError> {
5991        let Some(module) = self.supervisor.get(&module_id) else {
5992            return Ok(vec![control_error_frame(
5993                &frame,
5994                "unknown_module",
5995                format!("module_id '{module_id}' is not supervised"),
5996            )?]);
5997        };
5998
5999        let snapshot = module.stderr_tail(
6000            max_lines.map(|value| value as usize),
6001            max_bytes.map(|value| value as usize),
6002        );
6003
6004        let response = ClientControlResponse::SupervisorStderrTail {
6005            module_id,
6006            tail: StderrTail {
6007                capture: match snapshot.capture {
6008                    CaptureState::Captured => StderrCaptureState::Captured,
6009                    CaptureState::Incomplete { reason } => {
6010                        StderrCaptureState::Incomplete { reason }
6011                    }
6012                    CaptureState::NotCaptured { reason } => {
6013                        StderrCaptureState::NotCaptured { reason }
6014                    }
6015                },
6016                entries: snapshot
6017                    .entries
6018                    .into_iter()
6019                    .map(|entry| match entry {
6020                        TailEntry::Line {
6021                            text,
6022                            truncated,
6023                            at_ms,
6024                        } => StderrTailEntry::Line {
6025                            text,
6026                            truncated,
6027                            at_ms,
6028                        },
6029                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6030                    })
6031                    .collect(),
6032                dropped_lines: snapshot.dropped_lines,
6033            },
6034        };
6035        Ok(vec![control_response_body_frame(
6036            &frame,
6037            &response,
6038            "ClientControlResponse::SupervisorStderrTail",
6039        )?])
6040    }
6041
6042    async fn observe_control_wait<F: std::future::Future>(
6043        &self,
6044        ctx: &RouteCtx,
6045        target: Option<&str>,
6046        kind: &'static str,
6047        future: F,
6048    ) -> F::Output {
6049        self.counters
6050            .observe_reader_wait(
6051                || {
6052                    let module = self
6053                        .forwarding
6054                        .module_id_for_connection(ctx.connection_id)
6055                        .ok()
6056                        .flatten();
6057                    let principal = if module.is_some() { "module" } else { "direct" };
6058                    (
6059                        self.resolve_reader_wait_target(kind, target)
6060                            .or(module)
6061                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6062                        principal.into(),
6063                    )
6064                },
6065                kind,
6066                future,
6067            )
6068            .await
6069    }
6070
6071    /// Resolve the reader-wait module ID only from supervisor or registry
6072    /// records; never attribute a wait from the request selector alone.
6073    pub(crate) fn resolve_reader_wait_target(
6074        &self,
6075        op: &str,
6076        selector: Option<&str>,
6077    ) -> Option<String> {
6078        let selector = selector?;
6079        match op {
6080            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6081                .supervisor
6082                .get(selector)
6083                .map(|module| module.module_id().to_owned()),
6084            ops::SUPERVISOR_HEALTH_PROBE => self
6085                .registry
6086                .get_module(selector)
6087                .ok()
6088                .flatten()
6089                .map(|registration| registration.manifest.module_id),
6090            _ => None,
6091        }
6092    }
6093
6094    async fn handle_supervisor_terminals(
6095        &self,
6096        ctx: &RouteCtx,
6097        frame: Frame,
6098        module_id: String,
6099    ) -> Result<Vec<Frame>, RouterError> {
6100        let Some(module) = self.supervisor.get(&module_id) else {
6101            return Ok(vec![control_error_frame(
6102                &frame,
6103                "unknown_module",
6104                format!("module_id '{module_id}' is not supervised"),
6105            )?]);
6106        };
6107
6108        // The journal read runs on a blocking thread: it can be megabytes of
6109        // file I/O and must not occupy a runtime worker.
6110        let terminals = self
6111            .observe_control_wait(
6112                ctx,
6113                Some(module.module_id()),
6114                ops::SUPERVISOR_TERMINALS,
6115                module.read_durable_terminal_history(),
6116            )
6117            .await
6118            .map_err(|error| {
6119                RouterError::backend(
6120                    0,
6121                    frame.header.corr,
6122                    format!("failed to read terminal history: {error}"),
6123                )
6124            })?;
6125        let response = ClientControlResponse::SupervisorTerminals {
6126            module_id,
6127            terminals,
6128        };
6129        Ok(vec![control_response_body_frame(
6130            &frame,
6131            &response,
6132            "ClientControlResponse::SupervisorTerminals",
6133        )?])
6134    }
6135
6136    fn handle_supervisor_routes(
6137        &self,
6138        frame: Frame,
6139        module_id: Option<String>,
6140    ) -> Result<Vec<Frame>, RouterError> {
6141        let modules = self
6142            .forwarding
6143            .route_census(module_id.as_deref())
6144            .map_err(RouterError::Forwarding)?
6145            .into_iter()
6146            .map(|(module_id, routes)| SupervisorRouteModule {
6147                module_id,
6148                routes: routes
6149                    .into_iter()
6150                    .map(|route| SupervisorRoute {
6151                        consumer: match route.principal {
6152                            Principal::Reserved { module_id } => {
6153                                SupervisorRouteConsumer::Reserved { module_id }
6154                            }
6155                            Principal::Direct | Principal::Unverified => {
6156                                SupervisorRouteConsumer::Direct {
6157                                    connection_id: route.goodbye_target.connection_id.get(),
6158                                }
6159                            }
6160                        },
6161                        age_ms: Instant::now()
6162                            .saturating_duration_since(route.bound_at)
6163                            .as_millis()
6164                            .try_into()
6165                            .unwrap_or(u64::MAX),
6166                        draining: route.draining,
6167                        drain_reason: route.drain_reason,
6168                    })
6169                    .collect(),
6170            })
6171            .collect();
6172        let response = ClientControlResponse::SupervisorRoutes { modules };
6173        Ok(vec![control_response_body_frame(
6174            &frame,
6175            &response,
6176            "ClientControlResponse::SupervisorRoutes",
6177        )?])
6178    }
6179
6180    async fn handle_supervisor_provenance(
6181        &self,
6182        ctx: &RouteCtx,
6183        frame: Frame,
6184        module_id: Option<String>,
6185    ) -> Result<Vec<Frame>, RouterError> {
6186        let targeted = module_id.is_some();
6187        let selected = if let Some(module_id) = module_id {
6188            let Some(module) = self.supervisor.get(&module_id) else {
6189                return Ok(vec![control_error_frame(
6190                    &frame,
6191                    "unknown_module",
6192                    format!("module_id '{module_id}' is not supervised"),
6193                )?]);
6194            };
6195            vec![module]
6196        } else {
6197            self.supervisor.list()
6198        };
6199
6200        let target = targeted.then(|| selected[0].module_id());
6201        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6202            let mut modules = Vec::with_capacity(selected.len());
6203            for module in &selected {
6204                let (status, observed_image) = module
6205                    .status_and_running_image_agreement()
6206                    .await
6207                    .map_err(|err| {
6208                        RouterError::backend(
6209                            0,
6210                            frame.header.corr,
6211                            format!("failed to read supervisor status: {err}"),
6212                        )
6213                    })?;
6214                let module_declared = self
6215                    .registry
6216                    .get_module(&status.module_id)
6217                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6218                    .and_then(|registration| registration.manifest.provenance)
6219                    .map(|build| ModuleDeclaredProvenance::Reported { build })
6220                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6221                #[cfg(test)]
6222                let running_image = match &self.provenance_probe_override {
6223                    Some(result) => result.clone(),
6224                    None => observed_image,
6225                };
6226                #[cfg(not(test))]
6227                let running_image = observed_image;
6228                modules.push(SupervisorModuleProvenance {
6229                    module_id: status.module_id,
6230                    module_declared,
6231                    daemon_observed: SupervisorObservedProcess {
6232                        launch_nonce_source: status.launch_nonce_source,
6233                        pid: status.pid,
6234                        spawned_at_ms: status.spawned_at_ms,
6235                        spawned_from: status.spawned_from,
6236                        running_image,
6237                    },
6238                });
6239            }
6240            let daemon = SupervisorDaemonProvenance {
6241                daemon_build: self.daemon_provenance.build.clone(),
6242                daemon_observed: DaemonObservedProcess {
6243                    pid: self.daemon_provenance.pid,
6244                    started_at_ms: self
6245                        .daemon_provenance
6246                        .start_clock
6247                        .map(|clock| clock.started_at_ms())
6248                        .or(self.daemon_provenance.started_at_ms),
6249                    running_image: self
6250                        .daemon_provenance
6251                        .probe
6252                        .observe(
6253                            self.daemon_provenance.pid,
6254                            self.daemon_provenance.executable_path.as_deref(),
6255                            self.daemon_provenance.executable_identity.clone(),
6256                            self.daemon_provenance.process_start_time,
6257                        )
6258                        .await,
6259                },
6260            };
6261            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6262            Ok(vec![control_response_body_frame(
6263                &frame,
6264                &response,
6265                "ClientControlResponse::SupervisorProvenance",
6266            )?])
6267        })
6268        .await
6269    }
6270
6271    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6272        self.refresh_capability_requirements();
6273        let generation = self
6274            .registry
6275            .generation()
6276            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6277        let modules = self
6278            .supervisor
6279            .list()
6280            .into_iter()
6281            .map(|module| {
6282                let status = module.status_for_control("health").map_err(|err| {
6283                    RouterError::backend(
6284                        0,
6285                        frame.header.corr,
6286                        format!("failed to read supervisor health: {err}"),
6287                    )
6288                })?;
6289                let module_id = status.module_id;
6290                let capability_detail = self
6291                    .capability_evaluator
6292                    .required_problem_detail(&module_id);
6293                Ok(SupervisorHealthEntry {
6294                    module_id,
6295                    status: status.health.status,
6296                    detail: append_capability_problem_detail(
6297                        status.health.detail,
6298                        capability_detail,
6299                    ),
6300                    metrics: status.health.metrics,
6301                    consecutive_failures: status.health.consecutive_failures,
6302                    late_answer_count: status.health.late_answer_count,
6303                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6304                    last_action: status.health.last_action,
6305                    last_action_ms: status.health.last_action_ms,
6306                    last_probe_ms: status.health.last_probe_ms,
6307                })
6308            })
6309            .collect::<Result<Vec<_>, RouterError>>()?;
6310        let response = ClientControlResponse::SupervisorHealth {
6311            generation,
6312            modules,
6313        };
6314        Ok(vec![control_response_body_frame(
6315            &frame,
6316            &response,
6317            "ClientControlResponse::SupervisorHealth",
6318        )?])
6319    }
6320
6321    async fn handle_supervisor_restart(
6322        &self,
6323        frame: Frame,
6324        module_id: String,
6325        drain_timeout_ms: Option<u64>,
6326        requester: Principal,
6327    ) -> Result<Vec<Frame>, RouterError> {
6328        let operation_lock = self.supervisor.operation_lock();
6329        let _operation_guard = operation_lock.lock().await;
6330        let Some(module) = self.supervisor.get(&module_id) else {
6331            return Ok(vec![control_error_frame(
6332                &frame,
6333                "unknown_module",
6334                format!("module_id '{module_id}' is not supervised"),
6335            )?]);
6336        };
6337
6338        self.route_outages.mark_operator_action(&module_id);
6339        if let Err(err) = module
6340            .restart_requested(drain_timeout_ms, Some(requester))
6341            .await
6342        {
6343            self.route_outages
6344                .operator_action_ended_unrefused(&module_id);
6345            let (code, message) = match err {
6346                crate::supervise::SuperviseError::Disabled { .. } => {
6347                    ("module_disabled", err.to_string())
6348                }
6349                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6350                    ("swap_in_progress", err.to_string())
6351                }
6352                _ => (
6353                    "target_unavailable",
6354                    format!("failed to restart module_id '{module_id}': {err}"),
6355                ),
6356            };
6357            return Ok(vec![control_error_frame(&frame, code, message)?]);
6358        }
6359
6360        let response = ClientControlResponse::SupervisorAck {
6361            module_id,
6362            applied: true,
6363        };
6364        Ok(vec![control_response_body_frame(
6365            &frame,
6366            &response,
6367            "ClientControlResponse::SupervisorAck",
6368        )?])
6369    }
6370
6371    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
6372    /// when the old process has finished draining: a caller whose own lane
6373    /// rides the old process must get its reply before that drain waits on it.
6374    async fn handle_supervisor_swap(
6375        &self,
6376        frame: Frame,
6377        module_id: String,
6378        ready_timeout_ms: Option<u64>,
6379        requester: Principal,
6380    ) -> Result<Vec<Frame>, RouterError> {
6381        // The daemon-wide operation lock is held only to resolve the handle,
6382        // not across the swap. The swap can take its whole readiness budget,
6383        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6384        // holding it here would park an operator's stop behind the swap it is
6385        // meant to abort. A rescan or stop that reaches the module during the
6386        // swap is served by the swap itself (see `supervise_swap`).
6387        let module = {
6388            let operation_lock = self.supervisor.operation_lock();
6389            let _operation_guard = operation_lock.lock().await;
6390            self.supervisor.get(&module_id)
6391        };
6392        let Some(module) = module else {
6393            return Ok(vec![control_error_frame(
6394                &frame,
6395                "unknown_module",
6396                format!("module_id '{module_id}' is not supervised"),
6397            )?]);
6398        };
6399
6400        self.route_outages.mark_operator_action(&module_id);
6401        if let Err(err) = module
6402            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6403            .await
6404        {
6405            self.route_outages
6406                .operator_action_ended_unrefused(&module_id);
6407            use crate::supervise::SuperviseError;
6408            let message = err.to_string();
6409            let error = match err {
6410                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6411                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6412                    code: "swap_refused".to_string(),
6413                    message,
6414                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6415                },
6416                SuperviseError::SwapFailed {
6417                    arm,
6418                    candidate_exit,
6419                    ..
6420                } => ErrorBody {
6421                    code: "swap_failed".to_string(),
6422                    message,
6423                    detail: Some(serde_json::json!({
6424                        "arm": arm.as_str(),
6425                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6426                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6427                    })),
6428                },
6429                _ => ErrorBody::new(
6430                    "target_unavailable",
6431                    format!("failed to swap module_id '{module_id}': {message}"),
6432                ),
6433            };
6434            return Ok(vec![control_error_body_frame(&frame, error)?]);
6435        }
6436        // A completed swap kept the incumbent serving until cutover, so it
6437        // usually opened no outage; a mark left behind would make the next,
6438        // unrelated outage read as requested.
6439        self.route_outages
6440            .operator_action_ended_unrefused(&module_id);
6441
6442        let response = ClientControlResponse::SupervisorAck {
6443            module_id,
6444            applied: true,
6445        };
6446        Ok(vec![control_response_body_frame(
6447            &frame,
6448            &response,
6449            "ClientControlResponse::SupervisorAck",
6450        )?])
6451    }
6452
6453    async fn handle_supervisor_reload(
6454        &self,
6455        frame: Frame,
6456        module_id: String,
6457        requester: Principal,
6458    ) -> Result<Vec<Frame>, RouterError> {
6459        let operation_lock = self.supervisor.operation_lock();
6460        let _operation_guard = operation_lock.lock().await;
6461        let Some(module) = self.supervisor.get(&module_id) else {
6462            return Ok(vec![control_error_frame(
6463                &frame,
6464                "unknown_module",
6465                format!("module_id '{module_id}' is not supervised"),
6466            )?]);
6467        };
6468
6469        self.route_outages.mark_operator_action(&module_id);
6470        if let Err(err) = module.reload_requested(Some(requester)).await {
6471            self.route_outages
6472                .operator_action_ended_unrefused(&module_id);
6473            let (code, message) = match err {
6474                crate::supervise::SuperviseError::Disabled { .. } => {
6475                    ("module_disabled", err.to_string())
6476                }
6477                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6478                    ("swap_in_progress", err.to_string())
6479                }
6480                _ => (
6481                    "reload_failed",
6482                    format!("failed to reload module_id '{module_id}': {err}"),
6483                ),
6484            };
6485            return Ok(vec![control_error_frame(&frame, code, message)?]);
6486        }
6487
6488        let response = ClientControlResponse::SupervisorAck {
6489            module_id,
6490            applied: true,
6491        };
6492        Ok(vec![control_response_body_frame(
6493            &frame,
6494            &response,
6495            "ClientControlResponse::SupervisorAck",
6496        )?])
6497    }
6498
6499    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6500    /// the configuration comes from and whether to apply it; every mode then
6501    /// runs the same comparison, so a preview describes the apply it previews.
6502    async fn handle_supervisor_rescan(
6503        &self,
6504        frame: Frame,
6505        mode: RescanMode,
6506        requester: Principal,
6507    ) -> Result<Vec<Frame>, RouterError> {
6508        let preview = mode.previews();
6509        let Some(context) = self.rescan.clone() else {
6510            return Ok(vec![control_error_frame(
6511                &frame,
6512                "rescan_unavailable",
6513                "the daemon was not started with a reloadable config path".to_string(),
6514            )?]);
6515        };
6516
6517        let operation_lock = self.supervisor.operation_lock();
6518        let _operation_guard = operation_lock.lock().await;
6519        // Loaded under the operation lock, so no other rescan can change the
6520        // staged configuration between a checked apply's comparison and the
6521        // apply itself.
6522        let loaded = match self.load_rescan_config(&mode, &context) {
6523            Ok(config) => config,
6524            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6525        };
6526        // `load` reports a missing file as Ok(None), which is correct at boot
6527        // (no config, nothing to supervise) and catastrophic here: rescan treats
6528        // "not in the config" as "remove it", so an absent file would read as an
6529        // empty module list and retire the entire running fleet. An editor
6530        // writing via write-new-then-rename, or a half-finished edit, is enough
6531        // to open that window. Refuse instead: a config that cannot be read
6532        // carries no instruction to remove anything.
6533        let Some((config, activity_config)) = loaded else {
6534            return Ok(vec![control_error_frame(
6535                &frame,
6536                "invalid_daemon_config",
6537                format!(
6538                    "daemon config not found at {}; refusing to rescan (an absent config would \
6539                     retire every supervised module)",
6540                    context.config_path.display()
6541                ),
6542            )?]);
6543        };
6544        let (
6545            configured_port,
6546            storage_config,
6547            admission_facts_carrier_module_id,
6548            admission_facts_targets,
6549            scope_authority_owners,
6550            privileged_operator_prompt,
6551            modules,
6552            reserved_capabilities,
6553        ) = (
6554            config.port,
6555            config.storage,
6556            config.admission_facts_carrier_module_id,
6557            config.admission_facts_targets,
6558            config.scope_authority_owners,
6559            config.privileged_operator_prompt,
6560            config.modules,
6561            config.reserved_capabilities,
6562        );
6563
6564        // Collect the sections rescan cannot apply, so the REPLY carries them.
6565        //
6566        // The warning below has always been correct and has always gone only to
6567        // the journal -- addressed to whoever reads logs, while the person who
6568        // just edited the config is looking at the CLI. Naming each section
6569        // individually rather than setting a flag: "something outside modules
6570        // changed" sends the operator back to diffing their own file, which is
6571        // the work this is meant to save.
6572        let mut restart_required = Vec::new();
6573        for section in RestartRequiredSection::ALL {
6574            let changed = match section {
6575                RestartRequiredSection::Port => configured_port != context.configured_port,
6576                RestartRequiredSection::Storage => storage_config != context.storage_config,
6577                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6578                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6579                }
6580                RestartRequiredSection::AdmissionFactsTargets => {
6581                    admission_facts_targets != context.admission_facts_targets
6582                }
6583                RestartRequiredSection::ScopeAuthorityOwners => {
6584                    scope_authority_owners != context.scope_authority_owners
6585                }
6586                RestartRequiredSection::PrivilegedOperatorPrompt => {
6587                    privileged_operator_prompt != context.privileged_operator_prompt
6588                }
6589            };
6590            if changed {
6591                restart_required.push(section.label().to_string());
6592            }
6593        }
6594        if !restart_required.is_empty() {
6595            warn!(
6596                config_path = %context.config_path.display(),
6597                sections = %restart_required.join(", "),
6598                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6599            );
6600        }
6601
6602        for configured in &modules {
6603            if let Err(err) = validate_spec(&configured.module_spec()) {
6604                return Ok(vec![control_error_frame(
6605                    &frame,
6606                    "invalid_daemon_config",
6607                    format!("supervisor rescan rejected daemon config: {err}"),
6608                )?]);
6609            }
6610        }
6611
6612        let configured_capabilities = modules
6613            .iter()
6614            .map(|module| (module.module_id.clone(), module.enabled))
6615            .collect::<Vec<_>>();
6616        let preview_capability_warnings = if preview {
6617            let (_, registrations) = self.runtime_capability_snapshot()?;
6618            let current_modules = self
6619                .supervisor
6620                .list()
6621                .into_iter()
6622                .map(|module| module.module_id().to_string())
6623                .collect::<BTreeSet<_>>();
6624            let resulting_modules = configured_capabilities.clone();
6625            let removed = current_modules
6626                .into_iter()
6627                .filter(|module_id| {
6628                    !resulting_modules
6629                        .iter()
6630                        .any(|(configured_id, _)| configured_id == module_id)
6631                })
6632                .collect::<Vec<_>>();
6633            // A candidate preview also names providers the candidate disables.
6634            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6635            let disabled = match &mode {
6636                RescanMode::Candidate(_) => resulting_modules
6637                    .iter()
6638                    .filter(|(_, enabled)| !enabled)
6639                    .map(|(module_id, _)| module_id.clone())
6640                    .collect::<Vec<_>>(),
6641                _ => Vec::new(),
6642            };
6643            self.capability_evaluator.preview_warnings(
6644                resulting_modules,
6645                &removed,
6646                &disabled,
6647                &registrations,
6648            )
6649        } else {
6650            Vec::new()
6651        };
6652        let reconciled = match &mode {
6653            RescanMode::Candidate(candidate) => self
6654                .preview_candidate(&context, modules, candidate)
6655                .map(|result| (result, Vec::new())),
6656            _ => {
6657                self.reconcile_supervised_modules(
6658                    &context.supervisor,
6659                    modules,
6660                    preview,
6661                    requester.clone(),
6662                )
6663                .await
6664            }
6665        };
6666        let (result, changed) = match reconciled {
6667            Ok(result) => result,
6668            Err(message) => {
6669                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6670            }
6671        };
6672        if !preview {
6673            if let Some(ledger) = &self.activity {
6674                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6675                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6676            }
6677            if let Err(code) = self.configure_activity(activity_config).await {
6678                warn!(
6679                    code,
6680                    "activity retention could not be applied during rescan"
6681                );
6682            }
6683            self.capability_evaluator
6684                .configure(configured_capabilities, reserved_capabilities);
6685            self.capability_evaluator.wake_deadline_loop();
6686            self.refresh_capability_requirements();
6687        }
6688        let mut result = result;
6689        result.restart_required = restart_required;
6690        result.capability_warnings = preview_capability_warnings;
6691        let response = ClientControlResponse::SupervisorRescan { result };
6692        Ok(vec![control_response_body_frame(
6693            &frame,
6694            &response,
6695            "ClientControlResponse::SupervisorRescan",
6696        )?])
6697    }
6698
6699    async fn handle_supervisor_release_reserved(
6700        &self,
6701        frame: Frame,
6702        module_id: String,
6703    ) -> Result<Vec<Frame>, RouterError> {
6704        let Some(context) = self.rescan.clone() else {
6705            return Ok(vec![control_error_frame(
6706                &frame,
6707                "release_unavailable",
6708                "reserved-id release requires a daemon started with a reloadable config path",
6709            )?]);
6710        };
6711        let operation_lock = self.supervisor.operation_lock();
6712        let _operation_guard = operation_lock.lock().await;
6713        let loaded = match crate::daemon_config::load(&context.config_path) {
6714            Ok(Some(config)) => config,
6715            Ok(None) => {
6716                return Ok(vec![control_error_frame(
6717                    &frame,
6718                    "invalid_daemon_config",
6719                    format!(
6720                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6721                        context.config_path.display()
6722                    ),
6723                )?])
6724            }
6725            Err(err) => {
6726                return Ok(vec![control_error_frame(
6727                    &frame,
6728                    "invalid_daemon_config",
6729                    format!("unable to verify reserved-id release against daemon config: {err}"),
6730                )?])
6731            }
6732        };
6733        if loaded
6734            .modules
6735            .iter()
6736            .any(|configured| configured.module_id == module_id)
6737        {
6738            return Ok(vec![control_error_frame(
6739                &frame,
6740                "reserved_module_configured",
6741                format!(
6742                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6743                ),
6744            )?]);
6745        }
6746        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6747            return Ok(vec![control_error_frame(
6748                &frame,
6749                "reserved_gate_not_retained",
6750                format!(
6751                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6752                ),
6753            )?]);
6754        }
6755
6756        let response = ClientControlResponse::SupervisorAck {
6757            module_id,
6758            applied: true,
6759        };
6760        Ok(vec![control_response_body_frame(
6761            &frame,
6762            &response,
6763            "ClientControlResponse::SupervisorAck",
6764        )?])
6765    }
6766
6767    /// Compare the configured module set with the supervised one, changing
6768    /// nothing. The staged spec each supervised module holds is what the
6769    /// configured entry is compared with.
6770    fn plan_rescan(
6771        &self,
6772        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6773    ) -> Result<RescanPlan, String> {
6774        let mut current = BTreeMap::new();
6775        for module in self.supervisor.list() {
6776            let (spec, health) = module.configuration().map_err(|err| {
6777                format!(
6778                    "failed to read configuration for module_id '{}': {err}",
6779                    module.module_id()
6780                )
6781            })?;
6782            let enabled = module
6783                .status()
6784                .map_err(|err| {
6785                    format!(
6786                        "failed to read status for module_id '{}': {err}",
6787                        module.module_id()
6788                    )
6789                })?
6790                .enabled;
6791            current.insert(
6792                module.module_id().to_string(),
6793                (module, spec, health, enabled),
6794            );
6795        }
6796        let configured = configured_modules
6797            .into_iter()
6798            .map(|module| (module.module_id.clone(), module))
6799            .collect::<BTreeMap<_, _>>();
6800
6801        let added = configured
6802            .keys()
6803            .filter(|module_id| !current.contains_key(*module_id))
6804            .cloned()
6805            .collect::<Vec<_>>();
6806        let removed = current
6807            .keys()
6808            .filter(|module_id| !configured.contains_key(*module_id))
6809            .cloned()
6810            .collect::<Vec<_>>();
6811        let mut changed_pending_reload = Vec::new();
6812        let mut configuration_changes = BTreeSet::new();
6813        let mut enabled_changes = BTreeSet::new();
6814        let mut unchanged = 0_u32;
6815
6816        for (module_id, configured_module) in &configured {
6817            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6818            else {
6819                continue;
6820            };
6821            // Compare the whole launch spec so a future launch field cannot
6822            // accidentally become a live-only policy change. Health is stored
6823            // separately and applies live without replacing the process.
6824            let launch_changed = *current_spec != configured_module.module_spec();
6825            let configuration_changed =
6826                launch_changed || *current_health != configured_module.health;
6827            let enabled_changed = *current_enabled != configured_module.enabled;
6828            if configuration_changed {
6829                configuration_changes.insert(module_id.clone());
6830            }
6831            if launch_changed {
6832                changed_pending_reload.push(module_id.clone());
6833            }
6834            if enabled_changed {
6835                enabled_changes.insert(module_id.clone());
6836            }
6837            if !configuration_changed && !enabled_changed {
6838                unchanged = unchanged.saturating_add(1);
6839            }
6840        }
6841        Ok(RescanPlan {
6842            current,
6843            configured,
6844            added,
6845            removed,
6846            changed_pending_reload,
6847            configuration_changes,
6848            enabled_changes,
6849            unchanged,
6850        })
6851    }
6852
6853    /// Reconcile the running module set against the configured one.
6854    ///
6855    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6856    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6857    /// deliberately shares this function with the executing path rather than
6858    /// computing the same diff somewhere else -- two implementations of one
6859    /// decision agree until they do not, and the whole value of a preview is that
6860    /// it describes the operation that will actually run.
6861    async fn reconcile_supervised_modules(
6862        &self,
6863        supervisor: &Supervisor,
6864        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6865        preview: bool,
6866        requester: Principal,
6867    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6868        let plan = self.plan_rescan(configured_modules)?;
6869
6870        // Everything above this point is pure computation over two snapshots.
6871        // Everything below MUTATES. The preview returns here so the boundary is a
6872        // single early return rather than a condition repeated at each mutation
6873        // site, where one missed guard would apply part of a change the caller was
6874        // told would not happen.
6875        if preview {
6876            return Ok((plan.preview_result(), Vec::new()));
6877        }
6878        let changed = plan
6879            .configuration_changes
6880            .union(&plan.enabled_changes)
6881            .cloned()
6882            .collect::<Vec<_>>();
6883        let RescanPlan {
6884            current,
6885            configured,
6886            added,
6887            removed,
6888            changed_pending_reload,
6889            configuration_changes,
6890            enabled_changes,
6891            unchanged,
6892        } = plan;
6893        // The staged generation moves before the first change rather than
6894        // after the last, so a rescan that fails partway still invalidates
6895        // every earlier `supervisor.rescan_checked` preview: part of its diff
6896        // may have run.
6897        if !(added.is_empty()
6898            && removed.is_empty()
6899            && configuration_changes.is_empty()
6900            && enabled_changes.is_empty())
6901        {
6902            if let Some(context) = &self.rescan {
6903                context
6904                    .generation
6905                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6906            }
6907        }
6908
6909        for module_id in &removed {
6910            let module = &current
6911                .get(module_id)
6912                .expect("removed module came from current supervisor state")
6913                .0;
6914            module
6915                .retire_requested(Some(requester.clone()))
6916                .await
6917                .map_err(|err| {
6918                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6919                })?;
6920            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6921            //
6922            // `handle_route_open` resolves an absent module in three steps:
6923            // registry, then supervisor status, then tombstone. Retiring first
6924            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6925            // went with the teardown above, the supervisor entry went with
6926            // `retire`, and the tombstone does not exist yet -- so a route.open
6927            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6928            // it") for a module that was deliberately removed and whose caller
6929            // should get `module_removed` (TERMINAL, carrying a removal age).
6930            //
6931            // Writing the tombstone first closes it: during the window the
6932            // supervisor entry still answers, so the caller gets
6933            // `target_unavailable` -- retryable, and TRUE, because the module
6934            // is mid-teardown. After both statements it is `module_removed`.
6935            // No instant remains where a removed module reads as one that
6936            // never existed.
6937            //
6938            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6939            // the absence of a test beside a fix invites deletion: these are
6940            // two sync statements with no await between them, so reaching the
6941            // window needs a second worker thread to land exactly between them
6942            // and there is no hook to force it. MEASURED: the 25 daemon_config
6943            // tests pass identically with the old order and the new one, so
6944            // the existing suite cannot see this and a green run is not
6945            // evidence either way. What the suite does hold is the
6946            // post-condition -- a removed module answers `module_removed` --
6947            // which this preserves.
6948            //
6949            // Found by an Athena panel reading the shipped tree against a
6950            // design note (2026-09-19), as the one concrete instance of that
6951            // note's class that survived contact with source. Direction is
6952            // benign: retryable where terminal was intended, never the reverse.
6953            self.supervisor.record_rescan_removal(module_id);
6954            self.supervisor.retire(module_id);
6955            self.route_outages.forget(module_id);
6956        }
6957
6958        for module_id in configured.keys() {
6959            let Some((module, _, _, _)) = current.get(module_id) else {
6960                continue;
6961            };
6962            let configured_module = configured
6963                .get(module_id)
6964                .expect("configured module id came from configured map");
6965            if configuration_changes.contains(module_id) {
6966                module
6967                    .update_configuration(
6968                        configured_module.module_spec(),
6969                        configured_module.health.clone(),
6970                        configured_module.drain_timeout_ms,
6971                    )
6972                    .await
6973                    .map_err(|err| {
6974                        format!(
6975                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6976                        )
6977                    })?;
6978            }
6979            if enabled_changes.contains(module_id) {
6980                // A rescan that starts or stops a module applies an operator's
6981                // edit to the config, so the resulting outage was asked for.
6982                self.route_outages.mark_operator_action(module_id);
6983                module
6984                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6985                    .await
6986                    .map_err(|err| {
6987                        self.route_outages.operator_action_ended_unrefused(module_id);
6988                        format!(
6989                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6990                            configured_module.enabled
6991                        )
6992                    })?;
6993            }
6994        }
6995
6996        for module_id in &added {
6997            let configured_module = configured
6998                .get(module_id)
6999                .expect("added module id came from configured map");
7000            supervisor
7001                .supervise_configured_requested(
7002                    configured_module.module_spec(),
7003                    configured_module.enabled,
7004                    configured_module.health.clone(),
7005                    configured_module.drain_timeout_ms,
7006                    configured_module.restart,
7007                    configured_module.register_timeout,
7008                    Some(requester.clone()),
7009                )
7010                .map_err(|err| {
7011                    format!("failed to add module_id '{module_id}' during rescan: {err}")
7012                })?;
7013        }
7014
7015        Ok((
7016            SupervisorRescanResult {
7017                added,
7018                removed,
7019                changed_pending_reload,
7020                enabled_changes: enabled_changes.iter().cloned().collect(),
7021                unchanged,
7022                preview: false,
7023                // Filled by the caller, which is the only layer that can see the
7024                // previous config to diff against.
7025                restart_required: Vec::new(),
7026                capability_warnings: Vec::new(),
7027                module_actions: Vec::new(),
7028                earlier_pending: Vec::new(),
7029                expect: None,
7030            },
7031            changed,
7032        ))
7033    }
7034
7035    async fn handle_supervisor_set_enabled(
7036        &self,
7037        frame: Frame,
7038        module_id: String,
7039        enabled: bool,
7040        requester: Principal,
7041    ) -> Result<Vec<Frame>, RouterError> {
7042        let operation_lock = self.supervisor.operation_lock();
7043        let _operation_guard = operation_lock.lock().await;
7044        let Some(module) = self.supervisor.get(&module_id) else {
7045            return Ok(vec![control_error_frame(
7046                &frame,
7047                "unknown_module",
7048                format!("module_id '{module_id}' is not supervised"),
7049            )?]);
7050        };
7051
7052        // Enabling counts as well as disabling: a module an operator starts
7053        // is refused until it registers, and that wait was asked for.
7054        self.route_outages.mark_operator_action(&module_id);
7055        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7056            Ok(applied) => applied,
7057            Err(err) => {
7058                self.route_outages
7059                    .operator_action_ended_unrefused(&module_id);
7060                return Ok(vec![control_error_frame(
7061                    &frame,
7062                    "target_unavailable",
7063                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7064                )?]);
7065            }
7066        };
7067        if !applied {
7068            // Already in the requested state: nothing was made unavailable,
7069            // so the mark must not outlive this request.
7070            self.route_outages
7071                .operator_action_ended_unrefused(&module_id);
7072        }
7073
7074        self.capability_evaluator.wake_deadline_loop();
7075        self.refresh_capability_requirements();
7076        let response = ClientControlResponse::SupervisorAck { module_id, applied };
7077        Ok(vec![control_response_body_frame(
7078            &frame,
7079            &response,
7080            "ClientControlResponse::SupervisorAck",
7081        )?])
7082    }
7083
7084    async fn handle_supervisor_health_probe(
7085        &self,
7086        ctx: &RouteCtx,
7087        frame: Frame,
7088        module_id: String,
7089    ) -> Result<Vec<Frame>, RouterError> {
7090        self.refresh_capability_requirements();
7091        let Some(registration) = self
7092            .registry
7093            .get_module(&module_id)
7094            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7095        else {
7096            return Ok(vec![control_error_frame(
7097                &frame,
7098                "unknown_module",
7099                format!("module_id '{module_id}' is not registered"),
7100            )?]);
7101        };
7102
7103        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
7104        // named for it. Making `module_registration_grants_op` return false
7105        // unconditionally reddens five tests, and every one is named for something
7106        // else -- capability relay, probe/bind demultiplexing, supervision-only
7107        // probing. They exercise a successful advertisement check on the way to their
7108        // own subject.
7109        //
7110        // Real protection, fragile in a specific way: narrowing any of those tests to
7111        // focus on its stated subject would silently remove coverage nobody knows
7112        // they are carrying. Recorded here rather than as a sixth test, because the
7113        // useful fact is WHICH tests hold the guard up -- a new test would add
7114        // coverage without telling the next person what the existing ones quietly do.
7115        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7116        {
7117            return Ok(vec![control_error_frame(
7118                &frame,
7119                "health_not_advertised",
7120                format!("module_id '{module_id}' did not advertise health.check"),
7121            )?]);
7122        }
7123
7124        self.observe_control_wait(
7125            ctx,
7126            Some(&registration.manifest.module_id),
7127            ops::SUPERVISOR_HEALTH_PROBE,
7128            async {
7129                let deadline = Instant::now() + self.health_probe_timeout;
7130                let pending = match self.forwarding.begin_module_control_rpc_for(
7131                    &module_id,
7132                    MODULE_CONTROL_OP_HEALTH_CHECK,
7133                    deadline,
7134                ) {
7135                    Ok(pending) => pending,
7136                    Err(err) => {
7137                        return Ok(vec![control_error_frame(
7138                            &frame,
7139                            forwarding_error_code(&err),
7140                            err.to_string(),
7141                        )?])
7142                    }
7143                };
7144
7145                let PendingModuleControlRpc {
7146                    endpoint,
7147                    module_sink,
7148                    negotiated_ver,
7149                    corr: probe_corr,
7150                    receiver,
7151                } = pending;
7152                let mut guard =
7153                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7154                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7155                    .map_err(|err| {
7156                        RouterError::backend(
7157                            0,
7158                            frame.header.corr,
7159                            format!("failed to encode health.check request: {err}"),
7160                        )
7161                    })?;
7162                let probe_frame = Frame::build_with_version(
7163                    negotiated_ver,
7164                    FrameType::Request,
7165                    control_flags(),
7166                    0,
7167                    0,
7168                    probe_corr,
7169                    probe_body,
7170                )
7171                .map_err(RouterError::FrameBuild)?;
7172
7173                // Queue admission and the reply share one deadline, so a module that
7174                // stops reading cannot park the requesting connection's reader forever.
7175                match timeout_at(deadline, module_sink.send(probe_frame)).await {
7176                    Ok(Ok(())) => {}
7177                    Ok(Err(err)) => {
7178                        return Ok(vec![control_error_frame(
7179                            &frame,
7180                            "target_unavailable",
7181                            err.to_string(),
7182                        )?]);
7183                    }
7184                    Err(_) => {
7185                        return Ok(vec![control_error_frame(
7186                            &frame,
7187                            "module_timeout",
7188                            format!(
7189                                "module_id '{module_id}' did not answer health.check within {:?}",
7190                                self.health_probe_timeout
7191                            ),
7192                        )?]);
7193                    }
7194                }
7195
7196                match timeout_at(deadline, receiver).await {
7197                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7198                        guard.disarm();
7199                        let Some(report) = response.health_report() else {
7200                            return Ok(vec![control_error_frame(
7201                                &frame,
7202                                "invalid_control_body",
7203                                "health.check RPC returned a non-health response",
7204                            )?]);
7205                        };
7206                        // Metrics go out whole here. The supervisor's cached snapshot
7207                        // caps this blob (see truncate_health_metrics), and this path
7208                        // exists precisely to answer without that cap -- so applying it
7209                        // here would leave no way to see what the cached view drops.
7210                        let HealthReport {
7211                            status,
7212                            detail,
7213                            metrics,
7214                        } = report;
7215                        let capability_detail = self
7216                            .capability_evaluator
7217                            .required_problem_detail(&module_id);
7218                        let response = ClientControlResponse::SupervisorHealthProbe {
7219                            module_id,
7220                            status,
7221                            detail: append_capability_problem_detail(detail, capability_detail),
7222                            metrics,
7223                        };
7224                        Ok(vec![control_response_body_frame(
7225                            &frame,
7226                            &response,
7227                            "ClientControlResponse::SupervisorHealthProbe",
7228                        )?])
7229                    }
7230                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7231                        guard.disarm();
7232                        // The module refused the health probe with its own body, so
7233                        // the refusal is relayed without the daemon-origin flag
7234                        // (bit 6): a client must not read it as the daemon's own
7235                        // refusal, which it may treat as never forwarded.
7236                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7237                    }
7238                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7239                        guard.disarm();
7240                        Ok(vec![control_error_frame(
7241                            &frame,
7242                            "target_unavailable",
7243                            message,
7244                        )?])
7245                    }
7246                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7247                        guard.disarm();
7248                        Ok(vec![control_error_frame(
7249                            &frame,
7250                            "invalid_control_body",
7251                            message,
7252                        )?])
7253                    }
7254                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7255                        guard.disarm();
7256                        Ok(vec![control_error_frame(
7257                            &frame,
7258                            "invalid_control_body",
7259                            format!("expected module-control op '{expected}', got '{actual}'"),
7260                        )?])
7261                    }
7262                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7263                        guard.disarm();
7264                        Ok(vec![control_error_frame(
7265                            &frame,
7266                            "module_timeout",
7267                            format!(
7268                                "module_id '{module_id}' answered health.check after {:?}",
7269                                self.health_probe_timeout
7270                            ),
7271                        )?])
7272                    }
7273                    Ok(Err(_)) => Ok(vec![control_error_frame(
7274                        &frame,
7275                        "target_unavailable",
7276                        "health.check waiter was canceled before the module responded",
7277                    )?]),
7278                    Err(_) => Ok(vec![control_error_frame(
7279                        &frame,
7280                        "module_timeout",
7281                        format!(
7282                            "module_id '{module_id}' did not answer health.check within {:?}",
7283                            self.health_probe_timeout
7284                        ),
7285                    )?]),
7286                }
7287            },
7288        )
7289        .await
7290    }
7291
7292    fn supervisor_status(
7293        &self,
7294        module_id: &str,
7295        corr: u64,
7296    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7297        self.supervisor
7298            .get(module_id)
7299            .map(|module| {
7300                let warming = module.is_warming_for_control("status").map_err(|err| {
7301                    RouterError::backend(
7302                        0,
7303                        corr,
7304                        format!(
7305                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
7306                        ),
7307                    )
7308                })?;
7309                module.status_for_control("status").map_err(|err| {
7310                    RouterError::backend(
7311                        0,
7312                        corr,
7313                        format!(
7314                            "failed to read supervisor status for module_id '{module_id}': {err}"
7315                        ),
7316                    )
7317                }).map(|status| (status, warming))
7318            })
7319            .transpose()
7320    }
7321
7322    fn guard_module_control_op(
7323        &self,
7324        frame: &Frame,
7325        module_id: &str,
7326        op: &str,
7327    ) -> Result<Option<Frame>, RouterError> {
7328        if self.module_grants_op(module_id, op, frame.header.corr)? {
7329            return Ok(None);
7330        }
7331
7332        Ok(Some(control_error_frame(
7333            frame,
7334            "op_not_allowed",
7335            format!("module_id '{module_id}' did not grant control op '{op}'"),
7336        )?))
7337    }
7338
7339    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7340        let Some(registration) = self
7341            .registry
7342            .get_module(module_id)
7343            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7344        else {
7345            return Ok(false);
7346        };
7347        Ok(module_registration_grants_op(&registration.control_ops, op))
7348    }
7349
7350    fn handle_status_update(
7351        &self,
7352        endpoint: ModuleEndpointId,
7353        frame: Frame,
7354    ) -> Result<Vec<Frame>, RouterError> {
7355        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7356            Ok(update) => update,
7357            Err(err) => {
7358                // Forward-compat: a newer module may push a channel-0 op this subc
7359                // version doesn't know. The control contract says unknown push ops
7360                // are IGNORED, never answered with an error. Only a malformed body
7361                // for an op we DO know is a real error worth surfacing.
7362                if is_known_module_push_op(&frame.body) {
7363                    return Ok(vec![control_error_frame(
7364                        &frame,
7365                        "invalid_control_body",
7366                        format!("malformed module control push body: {err}"),
7367                    )?]);
7368                }
7369                return Ok(Vec::new());
7370            }
7371        };
7372
7373        match update {
7374            ModuleControlPush::RouteStatus {
7375                route_channel,
7376                route_epoch,
7377                status,
7378            } => {
7379                self.forwarding
7380                    .cache_status(endpoint, route_channel, route_epoch, status)
7381                    .map_err(RouterError::Forwarding)?;
7382            }
7383        }
7384        Ok(Vec::new())
7385    }
7386
7387    fn handle_route_poll(
7388        &self,
7389        ctx: &RouteCtx,
7390        frame: Frame,
7391        route_channel: u16,
7392        route_epoch: u32,
7393        kind: PollKind,
7394    ) -> Result<Vec<Frame>, RouterError> {
7395        let snapshot = self
7396            .forwarding
7397            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7398            .map_err(RouterError::Forwarding)?;
7399        let response = match (kind, snapshot) {
7400            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7401                ClientControlResponse::RoutePoll {
7402                    route_channel,
7403                    route_epoch,
7404                    status,
7405                    live: None,
7406                }
7407            }
7408            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7409                route_channel,
7410                route_epoch,
7411                status: None,
7412                live: None,
7413            },
7414            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7415                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7416                // is what makes reporting `true` correct rather than a
7417                // confident guess. `process_live` returns None only when the
7418                // module id has no supervisor snapshot at all -- an
7419                // externally-started module the daemon did not spawn -- and
7420                // for those the supervisor has no opinion to offer, ever. It
7421                // is never None for a supervised module in an unknown state:
7422                // a supervised module always has a snapshot, and the answer
7423                // comes from `state == Running && process_alive`.
7424                //
7425                // The route is Bound, so the module completed a HELLO on a
7426                // live connection; "the process this route points at is
7427                // running" is therefore attested by the binding rather than
7428                // assumed. Reporting `false` for an unsupervised module would
7429                // be the actual lie -- it would tell a client its healthy
7430                // route is dead because the daemon does not manage the
7431                // process.
7432                //
7433                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7434                // module whose liveness is genuinely unknown, e.g. a snapshot
7435                // that has not been populated yet -- THIS DEFAULT BECOMES
7436                // WRONG and must split: unsupervised stays true, unknown
7437                // becomes null so the client can tell the two apart. The
7438                // response field is already `Option<bool>`, so the wire can
7439                // carry that distinction today.
7440                let live = self
7441                    .process_liveness
7442                    .as_ref()
7443                    .and_then(|source| source.process_live(&module_id))
7444                    .unwrap_or(true);
7445                ClientControlResponse::RoutePoll {
7446                    route_channel,
7447                    route_epoch,
7448                    status: None,
7449                    live: Some(live),
7450                }
7451            }
7452            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7453                route_channel,
7454                route_epoch,
7455                status: None,
7456                live: Some(false),
7457            },
7458        };
7459
7460        Ok(vec![control_response_body_frame(
7461            &frame,
7462            &response,
7463            "ClientControlResponse::RoutePoll",
7464        )?])
7465    }
7466
7467    pub(crate) fn observe_module_control_completion(
7468        &self,
7469        completion: ModuleControlRpcCompletion,
7470    ) -> bool {
7471        match completion {
7472            ModuleControlRpcCompletion::Unknown => false,
7473            ModuleControlRpcCompletion::Settled => true,
7474            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7475                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7476                info!(
7477                    module_id = %module_id,
7478                    latency_ms,
7479                    "late health.check answer proves the module is alive"
7480                );
7481                match self
7482                    .supervisor
7483                    .record_late_health_answer(&module_id, latency_ms)
7484                {
7485                    Ok(true) => {}
7486                    Ok(false) => debug!(
7487                        module_id = %module_id,
7488                        latency_ms,
7489                        "late health.check answer has no active supervisor snapshot"
7490                    ),
7491                    Err(err) => warn!(
7492                        module_id = %module_id,
7493                        latency_ms,
7494                        error = %err,
7495                        "failed to record late health.check answer"
7496                    ),
7497                }
7498                true
7499            }
7500        }
7501    }
7502
7503    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7504    /// the module connection whose frame is being handled, or to the client that
7505    /// relay was opened for.
7506    ///
7507    /// This runs on the MODULE connection's frame handler, where returning `Err`
7508    /// ends that connection -- and a module connection carries every client's
7509    /// routes to that module, so ending it costs the whole fleet its tools.
7510    /// `ConnectionClosing` carries the id of the connection that is closing, and
7511    /// when that id is a CLIENT's, the condition is entirely about that one
7512    /// client's route.open. A client-scoped condition has no authority over a
7513    /// shared module connection, so it is logged and the single relay is dropped:
7514    /// the client is going away, and `complete_pending_relay` already removed the
7515    /// relay before failing, so there is nothing left to settle. Anything that
7516    /// relay still reserved is released by that client's own connection teardown,
7517    /// which is already under way -- that is what "closing" means.
7518    ///
7519    /// Every other failure is a statement about THIS connection and stays fatal:
7520    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7521    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7522    /// frames correctly.
7523    fn refuse_to_end_module_connection_for_a_client(
7524        &self,
7525        module_connection_id: ConnectionId,
7526        corr: u64,
7527        err: ForwardingError,
7528    ) -> Result<(), RouterError> {
7529        if let ForwardingError::ConnectionClosing { connection_id } = err {
7530            if connection_id != module_connection_id {
7531                warn!(
7532                    module_connection_id = module_connection_id.get(),
7533                    client_connection_id = connection_id.get(),
7534                    corr,
7535                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7536                );
7537                return Ok(());
7538            }
7539        }
7540        Err(RouterError::Forwarding(err))
7541    }
7542
7543    fn handle_module_relay_response(
7544        &self,
7545        connection_id: ConnectionId,
7546        frame: Frame,
7547    ) -> Result<Vec<Frame>, RouterError> {
7548        let mut secondary_error = None;
7549        let outcome = match frame.header.ty {
7550            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7551                Ok(probe) if probe.op == "route.bind" => {
7552                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7553                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7554                            RouteBindRelayOutcome::Accepted
7555                        }
7556                        Ok(other) => {
7557                            let message =
7558                                format!("route.bind response carried unexpected body: {other:?}");
7559                            secondary_error = Some(control_error_frame(
7560                                &frame,
7561                                "invalid_control_body",
7562                                message.clone(),
7563                            )?);
7564                            RouteBindRelayOutcome::ModuleGone(message)
7565                        }
7566                        Err(err) => {
7567                            let message = format!("malformed route.bind response body: {err}");
7568                            secondary_error = Some(control_error_frame(
7569                                &frame,
7570                                "invalid_control_body",
7571                                message.clone(),
7572                            )?);
7573                            RouteBindRelayOutcome::ModuleGone(message)
7574                        }
7575                    }
7576                }
7577                Ok(probe) => {
7578                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7579                    {
7580                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7581                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7582                            "malformed {} response body: {err}",
7583                            probe.op
7584                        )),
7585                    };
7586                    let completion = self
7587                        .forwarding
7588                        .complete_module_control_rpc(
7589                            connection_id,
7590                            frame.header.corr,
7591                            Some(&probe.op),
7592                            outcome,
7593                        )
7594                        .map_err(RouterError::Forwarding)?;
7595                    if !self.observe_module_control_completion(completion) {
7596                        debug!(
7597                            connection_id = connection_id.get(),
7598                            corr = frame.header.corr,
7599                            op = %probe.op,
7600                            "dropping late or unknown module-control RPC response"
7601                        );
7602                    }
7603                    return Ok(Vec::new());
7604                }
7605                Err(err) => {
7606                    if let Some(expected_op) = self
7607                        .forwarding
7608                        .pending_module_control_op(connection_id, frame.header.corr)
7609                        .map_err(RouterError::Forwarding)?
7610                    {
7611                        let completion = self
7612                            .forwarding
7613                            .complete_module_control_rpc(
7614                                connection_id,
7615                                frame.header.corr,
7616                                None,
7617                                ModuleControlRpcOutcome::MalformedResponse(format!(
7618                                    "malformed {expected_op} response body: {err}"
7619                                )),
7620                            )
7621                            .map_err(RouterError::Forwarding)?;
7622                        if !self.observe_module_control_completion(completion) {
7623                            debug!(
7624                                connection_id = connection_id.get(),
7625                                corr = frame.header.corr,
7626                                "dropping late malformed module-control RPC response"
7627                            );
7628                        }
7629                        return Ok(Vec::new());
7630                    }
7631                    let message = format!("malformed route.bind response body: {err}");
7632                    secondary_error = Some(control_error_frame(
7633                        &frame,
7634                        "invalid_control_body",
7635                        message.clone(),
7636                    )?);
7637                    RouteBindRelayOutcome::ModuleGone(message)
7638                }
7639            },
7640            FrameType::Error => {
7641                if self
7642                    .forwarding
7643                    .pending_module_control_op(connection_id, frame.header.corr)
7644                    .map_err(RouterError::Forwarding)?
7645                    .is_some()
7646                {
7647                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7648                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7649                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7650                            "malformed module-control ERROR body: {err}"
7651                        )),
7652                    };
7653                    let completion = self
7654                        .forwarding
7655                        .complete_module_control_rpc(
7656                            connection_id,
7657                            frame.header.corr,
7658                            None,
7659                            outcome,
7660                        )
7661                        .map_err(RouterError::Forwarding)?;
7662                    if !self.observe_module_control_completion(completion) {
7663                        debug!(
7664                            connection_id = connection_id.get(),
7665                            corr = frame.header.corr,
7666                            "dropping late or unknown module-control RPC error"
7667                        );
7668                    }
7669                    return Ok(Vec::new());
7670                }
7671                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7672                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7673                    Err(err) => {
7674                        let message = format!("malformed route.bind ERROR body: {err}");
7675                        secondary_error = Some(control_error_frame(
7676                            &frame,
7677                            "invalid_control_body",
7678                            message.clone(),
7679                        )?);
7680                        RouteBindRelayOutcome::ModuleGone(message)
7681                    }
7682                }
7683            }
7684            ty => {
7685                return Ok(vec![control_error_frame(
7686                    &frame,
7687                    "unsupported_control_frame",
7688                    format!("unsupported module channel-0 frame {ty:?}"),
7689                )?])
7690            }
7691        };
7692
7693        let settled =
7694            self.forwarding
7695                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7696        let completion = match settled {
7697            Ok(completion) => completion,
7698            Err(err) => {
7699                self.refuse_to_end_module_connection_for_a_client(
7700                    connection_id,
7701                    frame.header.corr,
7702                    err,
7703                )?;
7704                return Ok(secondary_error.into_iter().collect());
7705            }
7706        };
7707        if let Some(target) = completion.abandoned.as_ref() {
7708            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7709        }
7710        if !completion.settled {
7711            debug!(
7712                connection_id = connection_id.get(),
7713                corr = frame.header.corr,
7714                frame_type = ?frame.header.ty,
7715                "dropping late or unknown route.bind relay response"
7716            );
7717        }
7718        Ok(secondary_error.into_iter().collect())
7719    }
7720
7721    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7722        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7723        // GOODBYE ends the connection's logical session even when its socket
7724        // stays open. Use disconnect teardown so verdicts, client notices and
7725        // scope authority are released at the same lifecycle boundary.
7726        self.cleanup_connection_with_end_reason(
7727            connection_id,
7728            RegistrationEndReason::ExplicitGoodbye,
7729        )
7730        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7731        Ok(Vec::new())
7732    }
7733}
7734
7735impl Default for ControlHandler {
7736    fn default() -> Self {
7737        Self::new(Arc::new(Registry::default()))
7738    }
7739}
7740
7741impl crate::supervise::SwapPromotionObserver for ControlHandler {
7742    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7743        self.apply_registration_capabilities(registration);
7744    }
7745}
7746
7747fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7748    CapabilityRequirementStatus {
7749        consumer: status.consumer,
7750        capability: status.capability,
7751        need: match status.need {
7752            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7753            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7754        },
7755        verdict: status.verdict.as_str().to_string(),
7756        episode_seq: status.episode_seq,
7757        config_satisfiable: status.config_satisfiable,
7758        runtime_available: status.runtime_available,
7759        detail: status.detail,
7760    }
7761}
7762
7763fn append_capability_problem_detail(
7764    detail: Option<String>,
7765    capability_detail: Option<String>,
7766) -> Option<String> {
7767    match (detail, capability_detail) {
7768        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7769        (Some(detail), None) => Some(detail),
7770        (None, Some(capability_detail)) => Some(capability_detail),
7771        (None, None) => None,
7772    }
7773}
7774
7775fn subc_ops() -> Vec<String> {
7776    SUBC_CONTROL_OPS
7777        .iter()
7778        .map(|op| (*op).to_string())
7779        .collect()
7780}
7781
7782fn module_subc_ops() -> Vec<String> {
7783    SUBC_CONTROL_OPS
7784        .iter()
7785        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7786        .map(|op| (*op).to_string())
7787        .collect()
7788}
7789
7790#[cfg(test)]
7791fn module_baseline_control_ops() -> Vec<String> {
7792    MODULE_BASELINE_CONTROL_OPS
7793        .iter()
7794        .map(|op| (*op).to_string())
7795        .collect()
7796}
7797
7798fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7799    let mut seen = HashSet::new();
7800    let mut effective = Vec::new();
7801    for op in MODULE_BASELINE_CONTROL_OPS {
7802        if seen.insert((*op).to_string()) {
7803            effective.push((*op).to_string());
7804        }
7805    }
7806    for op in declared.unwrap_or_default() {
7807        if seen.insert(op.clone()) {
7808            effective.push(op);
7809        }
7810    }
7811    effective
7812}
7813
7814fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7815    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7816}
7817
7818fn target_module_id(target: &RouteTarget) -> &str {
7819    match target {
7820        RouteTarget::ToolProvider { module_id }
7821        | RouteTarget::ManagementSurface { module_id }
7822        | RouteTarget::InternalService { module_id, .. } => module_id,
7823    }
7824}
7825
7826fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7827    roles.iter().any(|role| match (target, role) {
7828        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7829        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7830        (
7831            RouteTarget::InternalService { service_id, .. },
7832            ProviderRole::InternalService {
7833                service_id: provided,
7834                ..
7835            },
7836        ) => service_id == provided,
7837        _ => false,
7838    })
7839}
7840
7841fn is_routable_role(role: &ProviderRole) -> bool {
7842    matches!(
7843        role,
7844        ProviderRole::ToolProvider { .. }
7845            | ProviderRole::ManagementSurface { .. }
7846            | ProviderRole::InternalService { .. }
7847    )
7848}
7849
7850#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7851enum ControlRequestBodyError {
7852    UnknownOp,
7853    InvalidBody,
7854}
7855
7856#[derive(Debug, Deserialize)]
7857struct ControlOpProbe {
7858    op: String,
7859}
7860
7861/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7862/// this set is treated as a forward-compat unknown and ignored rather than errored.
7863const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7864
7865fn is_known_module_push_op(body: &[u8]) -> bool {
7866    serde_json::from_slice::<ControlOpProbe>(body)
7867        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7868        .unwrap_or(false)
7869}
7870
7871fn is_known_module_request_op(body: &[u8]) -> bool {
7872    serde_json::from_slice::<ControlOpProbe>(body)
7873        .map(|probe| is_module_to_subc_op(&probe.op))
7874        .unwrap_or(false)
7875}
7876
7877fn is_module_to_subc_op(op: &str) -> bool {
7878    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7879}
7880
7881fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7882    debug!(
7883        op = %op,
7884        connection_id = connection_id.get(),
7885        corr,
7886        "control dispatch"
7887    );
7888}
7889
7890fn log_slow_control_dispatch(
7891    dispatch_started_at: Option<StdInstant>,
7892    op: &'static str,
7893    connection_id: ConnectionId,
7894    corr: u64,
7895    route_open_timings: Option<&RouteOpenTimings>,
7896) {
7897    let Some(dispatch_started_at) = dispatch_started_at else {
7898        return;
7899    };
7900    // Use the same monotonic clock as the phase timers and relay deadline.
7901    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7902    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7903        if let Some(timings) = route_open_timings {
7904            warn!(
7905                op = %op,
7906                connection_id = connection_id.get(),
7907                corr,
7908                elapsed_ms = elapsed.as_millis() as u64,
7909                module_id = ?timings.module_id,
7910                principal = %timings.principal,
7911                admission_ms = timings.admission.as_millis() as u64,
7912                project_root_ms = timings.project_root.as_millis() as u64,
7913                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7914                bind_send_ms = timings.bind_send.as_millis() as u64,
7915                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7916                "slow control dispatch"
7917            );
7918            return;
7919        }
7920        warn!(
7921            op = %op,
7922            connection_id = connection_id.get(),
7923            corr,
7924            elapsed_ms = elapsed.as_millis() as u64,
7925            "slow control dispatch"
7926        );
7927    }
7928}
7929
7930/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7931#[cfg(test)]
7932type TestCallerLookup =
7933    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7934
7935/// Every new client operation needs an explicit privilege decision here.
7936fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7937    match request {
7938        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7939            Some(PrivilegedOperation::Restart { module_id })
7940        }
7941        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7942            Some(PrivilegedOperation::Swap { module_id })
7943        }
7944        ClientControlRequest::SupervisorReload { module_id } => {
7945            Some(PrivilegedOperation::Reload { module_id })
7946        }
7947        ClientControlRequest::SupervisorRescan { preview } => {
7948            if *preview {
7949                None
7950            } else {
7951                Some(PrivilegedOperation::Rescan)
7952            }
7953        }
7954        // Only the well-formed apply asks for approval. A preview changes
7955        // nothing, and any other combination is refused before it touches
7956        // anything, so prompting the operator for it would ask about a request
7957        // that cannot run.
7958        ClientControlRequest::SupervisorRescanChecked {
7959            preview,
7960            candidate,
7961            expect,
7962        } => (!*preview && candidate.is_none() && expect.is_some())
7963            .then_some(PrivilegedOperation::Rescan),
7964        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7965            Some(PrivilegedOperation::SetEnabled {
7966                module_id,
7967                enabled: *enabled,
7968            })
7969        }
7970        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7971            Some(PrivilegedOperation::ReleaseReserved { module_id })
7972        }
7973        ClientControlRequest::ServerDescribe {}
7974        // The announcement uses the same client admission as supervisor.restart,
7975        // but cannot stop anything itself and needs no operator confirmation.
7976        | ClientControlRequest::DaemonShutdownIntent { .. }
7977        | ClientControlRequest::CatalogList { .. }
7978        | ClientControlRequest::RouteOpen { .. }
7979        | ClientControlRequest::RoutePoll { .. }
7980        | ClientControlRequest::SupervisorList {}
7981        | ClientControlRequest::SupervisorHealth {}
7982        | ClientControlRequest::SupervisorHealthProbe { .. }
7983        | ClientControlRequest::SupervisorRoutes { .. }
7984        | ClientControlRequest::SupervisorProvenance { .. }
7985        | ClientControlRequest::SupervisorStderrTail { .. }
7986        | ClientControlRequest::SupervisorTerminals { .. }
7987        | ClientControlRequest::SupervisorSpawnSnapshot {}
7988        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7989    }
7990}
7991
7992fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7993    match request {
7994        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
7995        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7996        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7997        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7998        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7999        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8000        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8001        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8002        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8003        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8004        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8005        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8006        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8007        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8008        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8009        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8010        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8011        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8012        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8013        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8014        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8015    }
8016}
8017
8018/// The caller of one `route.open`, as its `route.open refused` line names it.
8019///
8020/// A connection id alone cannot name a caller after the fact: ids restart at
8021/// every daemon restart, so the same number belongs to different modules
8022/// before and after one. These three fields are what the refused line needs
8023/// to be attributable on its own, and they are rendered with the same helpers
8024/// as the `route.open accepted` line so both directions grep alike.
8025struct RouteOpenCaller {
8026    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
8027    /// when the request presented a consumer identity whose launch nonce did
8028    /// not verify. The claimed module id is never printed as a principal.
8029    principal: String,
8030    /// `none` for an unscoped open; the admitted stamp's label once scope
8031    /// admission issued one; before that, the requested selector's label.
8032    scope: String,
8033    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
8034    /// because it is a string the caller chose.
8035    harness: String,
8036}
8037
8038impl RouteOpenCaller {
8039    fn new(
8040        principal: Option<&Principal>,
8041        scope: Option<&ScopeSelector>,
8042        identity: &BindIdentity,
8043    ) -> Self {
8044        Self {
8045            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8046            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8047            harness: bounded_log_text(&identity.harness),
8048        }
8049    }
8050
8051    /// For a refusal that has no parsed request at hand: names whatever the
8052    /// frame still says, and `unknown` where it says nothing parseable.
8053    fn unknown() -> Self {
8054        Self {
8055            principal: "unknown".to_string(),
8056            scope: "unknown".to_string(),
8057            harness: "unknown".to_string(),
8058        }
8059    }
8060}
8061
8062/// Longest client-chosen text (a scope ref, a harness) a route.open log line
8063/// prints before cutting it.
8064const MAX_LOGGED_TEXT_CHARS: usize = 64;
8065
8066/// `value` Debug-quoted, so control characters land escaped and cannot forge
8067/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
8068/// `…` inside the quotes so one request cannot flood the line.
8069fn bounded_log_text(value: &str) -> String {
8070    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8071    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8072        bounded.push('…');
8073    }
8074    format!("{bounded:?}")
8075}
8076
8077/// The `scope=` field of the `route.open accepted` line, from the stamp that
8078/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
8079/// so a ref holding control characters cannot forge a log line, and cut to 64
8080/// characters with a trailing `…` to bound the line. The bind proof and the
8081/// scope's attributes never appear.
8082fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8083    use subc_protocol::scope::ScopeKind;
8084    let kind = match stamp.kind {
8085        ScopeKind::Head => "head",
8086        ScopeKind::Worker => "worker",
8087        ScopeKind::Ephemeral => "ephemeral",
8088    };
8089    format!(
8090        "{kind}/{}/{}@{}",
8091        principal_label(&stamp.owner),
8092        bounded_log_text(&stamp.scope_ref),
8093        stamp.scope_epoch
8094    )
8095}
8096
8097/// The `scope=` field of a `route.open refused` line for an open refused
8098/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
8099/// The kind is not known yet, so the literal `requested` stands in its place.
8100/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
8101/// owner it has not been checked against the configured modules: a quoted
8102/// owner means the value was never validated. The bind proof never appears.
8103fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8104    let epoch = selector
8105        .scope_epoch
8106        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8107    format!(
8108        "requested/{}/{}@{epoch}",
8109        bounded_log_text(&principal_label(&selector.owner)),
8110        bounded_log_text(&selector.scope_ref),
8111    )
8112}
8113
8114fn principal_label(principal: &Principal) -> String {
8115    match principal {
8116        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8117        Principal::Direct => "direct".to_string(),
8118        other => format!("{other:?}"),
8119    }
8120}
8121
8122fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8123    match request {
8124        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8125        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8126        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8127        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8128        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8129    }
8130}
8131
8132fn parse_client_control_request(
8133    body: &[u8],
8134) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8135    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8136        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8137            Ok(probe)
8138                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8139                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8140            {
8141                ControlRequestBodyError::InvalidBody
8142            }
8143            Ok(_) => ControlRequestBodyError::UnknownOp,
8144            Err(_) => ControlRequestBodyError::InvalidBody,
8145        };
8146        (err, classification)
8147    })
8148}
8149
8150fn parse_module_control_request_from_module(
8151    body: &[u8],
8152) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8153    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8154        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8155            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8156            Ok(_) => ControlRequestBodyError::UnknownOp,
8157            Err(_) => ControlRequestBodyError::InvalidBody,
8158        };
8159        (err, classification)
8160    })
8161}
8162
8163#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8164enum ProviderRoleKind {
8165    ToolProvider,
8166    PipelineStage,
8167    ManagementSurface,
8168    InternalService,
8169}
8170
8171fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8172    match role {
8173        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8174        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8175        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8176        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8177    }
8178}
8179
8180fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8181    roles.iter().map(provider_role_kind).collect()
8182}
8183
8184/// Most refused scope records named individually in the log per sync; the
8185/// `refused` count on the accepted line is always complete.
8186const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8187
8188/// Per-outcome counts of one accepted `scope.sync`, for its log line.
8189#[derive(Debug, Default, PartialEq, Eq)]
8190struct ScopeOutcomeCounts {
8191    created: usize,
8192    replaced: usize,
8193    updated: usize,
8194    unchanged: usize,
8195    refused: usize,
8196}
8197
8198impl ScopeOutcomeCounts {
8199    fn of(results: &[ScopeRecordResult]) -> Self {
8200        let mut counts = Self::default();
8201        for result in results {
8202            let slot = match result.outcome {
8203                ScopeRecordOutcome::Created => &mut counts.created,
8204                ScopeRecordOutcome::Replaced => &mut counts.replaced,
8205                ScopeRecordOutcome::Updated => &mut counts.updated,
8206                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8207                ScopeRecordOutcome::Refused => &mut counts.refused,
8208            };
8209            *slot += 1;
8210        }
8211        counts
8212    }
8213}
8214
8215#[cfg(test)]
8216mod scope_outcome_count_tests {
8217    use super::*;
8218
8219    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8220        ScopeRecordResult {
8221            scope_ref: "r".to_string(),
8222            scope_epoch: 1,
8223            outcome,
8224            code: None,
8225            message: None,
8226            version: None,
8227            parent_state: None,
8228        }
8229    }
8230
8231    /// Each outcome lands in its own count, so a refused record can never be
8232    /// hidden inside the total the log already printed.
8233    #[test]
8234    fn every_outcome_is_counted_in_its_own_field() {
8235        let results = [
8236            result(ScopeRecordOutcome::Created),
8237            result(ScopeRecordOutcome::Created),
8238            result(ScopeRecordOutcome::Replaced),
8239            result(ScopeRecordOutcome::Updated),
8240            result(ScopeRecordOutcome::Unchanged),
8241            result(ScopeRecordOutcome::Refused),
8242            result(ScopeRecordOutcome::Refused),
8243            result(ScopeRecordOutcome::Refused),
8244        ];
8245        assert_eq!(
8246            ScopeOutcomeCounts::of(&results),
8247            ScopeOutcomeCounts {
8248                created: 2,
8249                replaced: 1,
8250                updated: 1,
8251                unchanged: 1,
8252                refused: 3,
8253            }
8254        );
8255    }
8256}
8257
8258/// Return whether a catalog change can create a newly violating live route.
8259/// Removing an attested claim is intentionally excluded: it makes fewer routes
8260/// forbidden and therefore must leave the existing route census untouched.
8261fn capability_census_trigger(
8262    old: Option<&CapabilityDeclarations>,
8263    new: Option<&CapabilityDeclarations>,
8264) -> bool {
8265    let old_provides = old
8266        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8267        .unwrap_or_default();
8268    let old_denies = old
8269        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8270        .unwrap_or_default();
8271    let new = new.cloned().unwrap_or(CapabilityDeclarations {
8272        provides: Vec::new(),
8273        requires: Vec::new(),
8274        must_never_reach: Vec::new(),
8275    });
8276
8277    new.provides
8278        .iter()
8279        .any(|capability| !old_provides.contains(capability))
8280        || new
8281            .must_never_reach
8282            .iter()
8283            .any(|capability| !old_denies.contains(capability))
8284}
8285
8286/// Find the first capability an attested opener denies that an attested target
8287/// claims. Both manifests are live registry records, never cached or client data.
8288fn denied_capability<'a>(
8289    opening_manifest: &'a ModuleManifest,
8290    target_manifest: &ModuleManifest,
8291) -> Option<&'a str> {
8292    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8293    let target_capabilities = target_manifest.capabilities.as_ref()?;
8294    opening_capabilities
8295        .must_never_reach
8296        .iter()
8297        .find(|denied| {
8298            target_capabilities
8299                .provides
8300                .iter()
8301                .any(|provided| provided == *denied)
8302        })
8303        .map(String::as_str)
8304}
8305
8306fn catalog_update_frozen_field_message(
8307    registered: &ModuleManifest,
8308    provides: &[ProviderRole],
8309) -> Option<String> {
8310    let old_has_provides = !registered.provides.is_empty();
8311    let new_has_provides = !provides.is_empty();
8312    if old_has_provides != new_has_provides {
8313        return Some(format!(
8314            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8315            registered.module_id
8316        ));
8317    }
8318
8319    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
8320        return Some(format!(
8321            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8322            registered.module_id
8323        ));
8324    }
8325
8326    let registered_concurrency = manifest_concurrency(registered);
8327    let mut candidate = registered.clone();
8328    candidate.provides = provides.to_vec();
8329    let candidate_concurrency = manifest_concurrency(&candidate);
8330    if candidate_concurrency != registered_concurrency {
8331        return Some(format!(
8332            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8333            registered.module_id, registered_concurrency, candidate_concurrency
8334        ));
8335    }
8336
8337    // control_ops live beside the manifest in the HELLO body, not inside
8338    // ModuleManifest, so a provides-only catalog.update cannot change them.
8339    None
8340}
8341
8342fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8343    manifest.provides.iter().any(is_routable_role)
8344}
8345
8346/// Returns the routable-provider concurrency subc should enforce for this manifest.
8347///
8348/// ToolProvider and ManagementSurface store their delivery concurrency directly.
8349/// InternalService has no role-specific concurrency field, so it retains the
8350/// existing ModuleManaged default for backward compatibility.
8351fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8352    manifest
8353        .provides
8354        .iter()
8355        .find_map(|provider| match provider {
8356            ProviderRole::ToolProvider { concurrency, .. }
8357            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8358            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8359        })
8360        .unwrap_or(Concurrency::ModuleManaged)
8361}
8362
8363/// True when the manifest carries a ManagementSurface role whose concurrency
8364/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
8365/// bytes because the typed manifest deliberately erases that distinction: the
8366/// default exists for wire compatibility, and this probe exists so the default
8367/// stays observable. Any parse irregularity returns false -- the caller only
8368/// logs, and a malformed body already failed registration upstream.
8369fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8370    let has_management_surface = manifest
8371        .provides
8372        .iter()
8373        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8374    if !has_management_surface {
8375        return false;
8376    }
8377    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8378        return false;
8379    };
8380    let Some(provides) = raw
8381        .get("manifest")
8382        .and_then(|manifest| manifest.get("provides"))
8383        .and_then(serde_json::Value::as_array)
8384    else {
8385        return false;
8386    };
8387    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
8388    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
8389    // against the management_surface_manifest_without_concurrency golden, not
8390    // recalled (the externally-tagged guess was this function's first bug).
8391    provides.iter().any(|role| {
8392        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8393            && role.get("concurrency").is_none()
8394    })
8395}
8396
8397fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8398    if peer_version != PROTOCOL_VERSION {
8399        return Err(format!(
8400            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8401        ));
8402    }
8403    Ok(PROTOCOL_VERSION)
8404}
8405
8406fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8407    Frame::build_with_version(
8408        response_version(frame),
8409        FrameType::Pong,
8410        frame.header.flags,
8411        0,
8412        0,
8413        frame.header.corr,
8414        Vec::new(),
8415    )
8416    .map_err(RouterError::FrameBuild)
8417}
8418
8419pub(crate) fn control_error_frame(
8420    frame: &Frame,
8421    code: &'static str,
8422    message: impl Into<String>,
8423) -> Result<Frame, RouterError> {
8424    control_error_body_frame(
8425        frame,
8426        ErrorBody {
8427            code: code.to_string(),
8428            message: message.into(),
8429            detail: None,
8430        },
8431    )
8432}
8433
8434/// A channel-0 ERROR for a refusal the daemon decided itself: every
8435/// route.open refusal, the drain `module_reloading` and scope-change/end
8436/// bodies, spawn-subscription and supervisor refusals. Carries the
8437/// daemon-origin bit, which is what lets a client treat `module_reloading`
8438/// as proof that its request never reached a module. Response, push,
8439/// goodbye and relay-request frames keep the plain control flags in their
8440/// own builders.
8441fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8442    control_error_frame_with_flags(frame, error, daemon_error_flags())
8443}
8444
8445/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8446/// refused route.bind answered as a route.open refusal, or a refused
8447/// module-control RPC. Plain control flags — the code and message are
8448/// module-chosen, so the frame must not claim daemon authorship, even when
8449/// the module picks a daemon code like `module_reloading`.
8450fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8451    control_error_frame_with_flags(frame, error, control_flags())
8452}
8453
8454fn control_error_frame_with_flags(
8455    frame: &Frame,
8456    error: ErrorBody,
8457    flags: Flags,
8458) -> Result<Frame, RouterError> {
8459    let body = serde_json::to_vec(&error).map_err(|err| {
8460        RouterError::backend(
8461            0,
8462            frame.header.corr,
8463            format!("failed to encode control ERROR: {err}"),
8464        )
8465    })?;
8466
8467    Frame::build_with_version(
8468        response_version(frame),
8469        FrameType::Error,
8470        flags,
8471        0,
8472        0,
8473        frame.header.corr,
8474        body,
8475    )
8476    .map_err(RouterError::FrameBuild)
8477}
8478
8479pub(crate) fn control_response_body_frame<T: Serialize>(
8480    frame: &Frame,
8481    reply: &T,
8482    label: &'static str,
8483) -> Result<Frame, RouterError> {
8484    let body = serde_json::to_vec(reply).map_err(|err| {
8485        RouterError::backend(
8486            0,
8487            frame.header.corr,
8488            format!("failed to encode {label}: {err}"),
8489        )
8490    })?;
8491
8492    Frame::build_with_version(
8493        response_version(frame),
8494        FrameType::Response,
8495        control_flags(),
8496        0,
8497        0,
8498        frame.header.corr,
8499        body,
8500    )
8501    .map_err(RouterError::FrameBuild)
8502}
8503
8504/// Map a forwarding failure to the wire code a client sees.
8505///
8506/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8507/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8508/// chosen here decides whether a caller retries or gives up.
8509///
8510/// That makes attribution the load-bearing property, not merely having a code. A
8511/// permanent fault published as a retryable one produces a fleet-wide retry storm
8512/// against something that can never recover; a transient fault published as
8513/// permanent gives up on work that would have succeeded. Both look correct in a
8514/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8515/// the mapping per variant rather than merely asserting that some code exists.
8516///
8517/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8518/// two codes on the same side of the boundary passes it. Measured rather than
8519/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8520/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8521/// a test named for something else that happens to assert the string.
8522///
8523/// That accidental coverage is deliberately left alone rather than promoted to a
8524/// named test, because it guards a property this function does not promise.
8525/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8526/// specific code within a class, so identity is free to change and only the
8527/// partition is a contract. Splitting it out would assert a guarantee nothing
8528/// depends on — and a suite that promises more than the code does is the harder
8529/// thing to correct later, because the next reader cannot tell which assertions
8530/// are load-bearing.
8531///
8532/// Pin identity here the moment a consumer branches on a specific code.
8533fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8534    match err {
8535        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8536        ForwardingError::NoModuleConnection => "target_unavailable",
8537        ForwardingError::ModuleReloading { .. } => "module_reloading",
8538        ForwardingError::ClientRouteChannelExhausted { .. }
8539        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8540        ForwardingError::StaleModuleEndpoint
8541        | ForwardingError::UnknownReservation { .. }
8542        | ForwardingError::ConnectionClosing { .. }
8543        | ForwardingError::ClientEgressClosed { .. }
8544        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8545        // Only a swap candidate's registration can produce this, and it means
8546        // exactly what a second active HELLO for a live id means.
8547        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8548        ForwardingError::RelayCorrelationExhausted
8549        | ForwardingError::RouteOpenBuild(_)
8550        | ForwardingError::Poisoned => "forwarding_error",
8551    }
8552}
8553
8554fn response_version(frame: &Frame) -> u8 {
8555    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8556        frame.header.ver
8557    } else {
8558        PROTOCOL_VERSION
8559    }
8560}
8561
8562fn control_flags() -> Flags {
8563    Flags::new(false, Priority::Passive, false)
8564}
8565
8566/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8567/// channel. The target is the module (a client never saw the route), so this
8568/// takes the module path: delivered late rather than dropped when the module's
8569/// queue is momentarily full, and never closing its connection.
8570fn send_goodbye_target_best_effort(
8571    counters: &DaemonCounters,
8572    target: &GoodbyeTarget,
8573    context: &'static str,
8574) {
8575    let Ok(frame) = Frame::build_with_version(
8576        target.negotiated_ver,
8577        FrameType::Goodbye,
8578        control_flags(),
8579        target.channel,
8580        target.epoch,
8581        0,
8582        Vec::new(),
8583    ) else {
8584        return;
8585    };
8586    crate::forwarding::send_module_route_goodbye(
8587        counters,
8588        &target.sink,
8589        frame,
8590        target.module_id.as_deref(),
8591        context,
8592    );
8593}
8594
8595pub(crate) fn send_route_control_pushes(
8596    forwarding: &ForwardingTable,
8597    routes: Vec<EndpointRoute>,
8598    push: ClientControlPush,
8599) {
8600    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8601    for route in routes {
8602        let target = route.goodbye_target;
8603        if let Some((existing, channels)) = targets
8604            .iter_mut()
8605            .find(|(existing, _)| existing.connection_id == target.connection_id)
8606        {
8607            debug_assert_eq!(
8608                existing.negotiated_ver, target.negotiated_ver,
8609                "one connection cannot negotiate multiple frame versions"
8610            );
8611            if !channels.contains(&target.channel) {
8612                channels.push(target.channel);
8613            }
8614            continue;
8615        }
8616        let channel = target.channel;
8617        targets.push((target, vec![channel]));
8618    }
8619    for (target, mut channels) in targets {
8620        channels.sort_unstable();
8621        let mut push = push.clone();
8622        match &mut push {
8623            ClientControlPush::RouteClosing {
8624                channels: covered, ..
8625            }
8626            | ClientControlPush::RouteClosed {
8627                channels: covered, ..
8628            } => *covered = channels,
8629        }
8630        let body = match serde_json::to_vec(&push) {
8631            Ok(body) => body,
8632            Err(err) => {
8633                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8634                continue;
8635            }
8636        };
8637        let frame = match Frame::build_with_version(
8638            target.negotiated_ver,
8639            FrameType::Push,
8640            control_flags(),
8641            0,
8642            0,
8643            0,
8644            body.clone(),
8645        ) {
8646            Ok(frame) => frame,
8647            Err(err) => {
8648                warn!(
8649                    route_channel = target.channel,
8650                    error = %err,
8651                    "failed to build route lifecycle control PUSH frame"
8652                );
8653                continue;
8654            }
8655        };
8656        if let Err(err) = target.sink.try_send(frame) {
8657            if target.close_on_delivery_failure() {
8658                warn!(
8659                    target_connection_id = target.connection_id.get(),
8660                    route_channel = target.channel,
8661                    error = %err,
8662                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8663                );
8664                let _ = forwarding.escalate_client_delivery_failure(
8665                    target.connection_id,
8666                    target.channel,
8667                    target.epoch,
8668                    CloseReason::new(
8669                        "route_lifecycle_push_delivery_failed",
8670                        format!(
8671                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8672                            target.channel
8673                        ),
8674                    ),
8675                    crate::forwarding::UndeliveredFrame {
8676                        module_id: target.module_id.as_deref(),
8677                        sink: &target.sink,
8678                    },
8679                );
8680            }
8681        }
8682    }
8683}
8684
8685#[cfg(test)]
8686mod tests {
8687    #[tokio::test]
8688    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8689        let registry = Arc::new(Registry::default());
8690        let handler = ControlHandler::new(registry);
8691        assert_eq!(
8692            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8693            None
8694        );
8695        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8696        hello_via_sink(
8697            &handler,
8698            &ctx,
8699            &mut rx,
8700            hello_frame("daemon", PROTOCOL_VERSION, 1),
8701        )
8702        .await;
8703        assert_eq!(
8704            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8705            Some("daemon".into())
8706        );
8707        assert_eq!(
8708            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8709            None
8710        );
8711        assert_eq!(
8712            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8713            None,
8714            "a registered module is not necessarily supervised"
8715        );
8716        assert_eq!(
8717            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8718            None,
8719            "aggregate provenance has no single target"
8720        );
8721        assert_eq!(
8722            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8723            None
8724        );
8725    }
8726
8727    mod peer_manifest {
8728        include!("peer_manifest_tests.rs");
8729    }
8730    mod activity_read {
8731        include!("activity_read_tests.rs");
8732    }
8733
8734    mod lifecycle_activity {
8735        include!("control_lifecycle_tests.rs");
8736    }
8737    #[cfg(unix)]
8738    #[tokio::test]
8739    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8740        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8741        let path = dir.join("subc.jsonc");
8742        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8743            "program":"/bin/sleep","args":["60"],"protocol":"none",
8744            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8745        }}}).to_string()).unwrap();
8746        let mut configured = crate::daemon_config::load(&path)
8747            .unwrap()
8748            .unwrap()
8749            .modules
8750            .pop()
8751            .unwrap();
8752        let registry = std::sync::Arc::new(crate::Registry::default());
8753        let handle = crate::SupervisorHandle::new();
8754        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8755            .with_handle(handle.clone());
8756        let module = supervisor
8757            .supervise_configured_with_health(
8758                configured.module_spec(),
8759                true,
8760                configured.health.clone(),
8761                None,
8762                configured.restart,
8763            )
8764            .unwrap();
8765        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8766        let before = module.status().unwrap().pid;
8767        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8768        configured.health.cadence = std::time::Duration::from_secs(3600);
8769        let health_only = handler
8770            .reconcile_supervised_modules(
8771                &supervisor,
8772                vec![configured.clone()],
8773                false,
8774                Principal::Direct,
8775            )
8776            .await
8777            .unwrap();
8778        assert!(
8779            health_only.0.changed_pending_reload.is_empty(),
8780            "health policy is already applied live"
8781        );
8782        assert_eq!(module.status().unwrap().pid, before);
8783        assert_eq!(
8784            module.configuration().unwrap().1.http,
8785            configured.health.http
8786        );
8787        configured.args = vec!["61".into()];
8788        let launch = handler
8789            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8790            .await
8791            .unwrap();
8792        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8793        assert_eq!(
8794            module.status().unwrap().pid,
8795            before,
8796            "a launch edit is stored until reload"
8797        );
8798        module.drain().await.unwrap();
8799    }
8800    use cortexkit_test_support::ScratchDir;
8801    use std::{
8802        collections::BTreeMap,
8803        fmt,
8804        path::PathBuf,
8805        sync::{Arc, Mutex},
8806        time::Duration,
8807    };
8808
8809    use serde_json::{json, Value};
8810    use subc_protocol::{
8811        manifest::{
8812            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8813            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8814        },
8815        session::HealthStatus,
8816        FrameType,
8817    };
8818
8819    use super::*;
8820    use crate::{
8821        forwarding::{DataRoute, DataRouteState},
8822        registry::ChannelState,
8823        router::FrameSink,
8824        stderr_tail::DEFAULT_MAX_LINE_BYTES,
8825        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8826        RouteCtx, Router,
8827    };
8828    use tokio::{
8829        sync::mpsc,
8830        time::{sleep, Instant},
8831    };
8832    use tracing::{
8833        field::{Field, Visit},
8834        Event, Subscriber,
8835    };
8836    use tracing_subscriber::{layer::Context, prelude::*, Layer};
8837
8838    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
8839    ///
8840    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
8841    /// is only populated for `tests/*.rs` integration test binaries -- this file
8842    /// compiles as part of the library target, which gets neither. This test's
8843    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
8844    /// and the sibling binary lives two directories up at
8845    /// `<target-dir>/<profile>/fake-aft-stub`.
8846    ///
8847    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
8848    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
8849    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
8850    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
8851    /// `NotFound`, which reads as a broken test rather than an unbuilt
8852    /// dependency -- so state the cause and the remedy instead. Deliberately a
8853    /// panic and not a silent skip: a test that quietly passes when it could not
8854    /// run is worse than one that fails, because it reports health it never
8855    /// verified.
8856    fn fake_aft_stub_path() -> PathBuf {
8857        let mut path = std::env::current_exe().expect("current_exe available in tests");
8858        path.pop(); // .../deps/
8859        path.pop(); // .../<profile>/
8860        path.push(if cfg!(windows) {
8861            "fake-aft-stub.exe"
8862        } else {
8863            "fake-aft-stub"
8864        });
8865        assert!(
8866            path.exists(),
8867            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8868             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8869            path.display()
8870        );
8871        path
8872    }
8873
8874    /// Whether clients retry `code` in place: the predicate itself, never a copy
8875    /// of its set. A copied list breaks silently when a code is added to or
8876    /// removed from the real one, and a stale copy here would let exactly the
8877    /// failure this test exists to catch pass.
8878    fn client_retries(code: &str) -> bool {
8879        subc_protocol::error_codes::is_retryable_route_open(code)
8880    }
8881
8882    /// A code is not a label — clients branch on it, so publishing the wrong KIND
8883    /// of failure is worse than publishing none. A permanent fault dressed as
8884    /// retryable makes every client in the fleet retry forever against something
8885    /// that cannot recover; a transient fault dressed as permanent abandons work
8886    /// that would have succeeded.
8887    ///
8888    /// Asserting "a code exists" cannot catch either, because the string is free
8889    /// to say anything. This enumerates every variant and pins which side of the
8890    /// retry boundary it lands on, so a new variant must be classified here
8891    /// deliberately rather than inheriting whichever arm it was appended to.
8892    #[test]
8893    fn retryability_of_forwarding_codes_matches_the_failure() {
8894        // Transient by nature: the target is booting, reloading, or its endpoint
8895        // was swapped mid-flight. Retrying is how these resolve.
8896        let transient = [
8897            ForwardingError::NoModuleConnection,
8898            ForwardingError::ModuleReloading {
8899                module_id: "m".into(),
8900            },
8901            ForwardingError::StaleModuleEndpoint,
8902            ForwardingError::UnknownReservation {
8903                client_channel: 1,
8904                module_channel: 1,
8905            },
8906            ForwardingError::ConnectionClosing {
8907                connection_id: ConnectionId::new(1),
8908            },
8909            ForwardingError::ClientEgressClosed {
8910                connection_id: ConnectionId::new(1),
8911            },
8912            ForwardingError::ModuleEgressUnavailable {
8913                connection_id: ConnectionId::new(1),
8914            },
8915        ];
8916        for err in transient {
8917            let code = forwarding_error_code(&err);
8918            assert!(
8919                client_retries(code),
8920                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8921            );
8922        }
8923
8924        // Not fixed by retrying. Channel and correlation exhaustion need the
8925        // caller to close routes, and a poisoned lock is a daemon that cannot
8926        // recover at all — the worst thing to advertise as retryable, since every
8927        // client would storm a daemon that will never answer.
8928        let permanent = [
8929            ForwardingError::ConnectionRoleConflict {
8930                connection_id: ConnectionId::new(1),
8931            },
8932            ForwardingError::ClientRouteChannelExhausted {
8933                connection_id: ConnectionId::new(1),
8934            },
8935            ForwardingError::ModuleRouteChannelExhausted {
8936                endpoint: ModuleEndpointId {
8937                    connection_id: ConnectionId::new(1),
8938                    generation: 1,
8939                },
8940            },
8941            ForwardingError::RelayCorrelationExhausted,
8942            ForwardingError::RouteOpenBuild("x".into()),
8943            ForwardingError::Poisoned,
8944        ];
8945        for err in permanent {
8946            let code = forwarding_error_code(&err);
8947            assert!(
8948                !client_retries(code),
8949                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8950            );
8951        }
8952    }
8953
8954    /// The principal is the daemon's answer to "who is calling", and modules
8955    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8956    /// plexus gates connector invocation. So a stamp is an authorization input in
8957    /// another process, not a label — and both possible answers SUCCEED, which is
8958    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8959    /// first-party capability to something that never proved it; a supervised one
8960    /// stamped `Direct` silently strips a module of capability it is entitled to.
8961    ///
8962    /// Neither shows up in a test that only checks the bind succeeded. Before this
8963    /// test the only coverage was accidental —
8964    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8965    /// stamped principal on its way past, so narrowing that wire-shape test to its
8966    /// stated subject would have deleted the last assertion on this value. It
8967    /// still asserts the stamp, which is now redundancy rather than the only
8968    /// guard: both fail under the same mutation, and this one names the reason.
8969    /// SCOPE: this handler's supervisor has spawned nothing, so
8970    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8971    /// is unreachable here. Both assertions below are refusals, and a mutant that
8972    /// refuses everything would satisfy them.
8973    ///
8974    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8975    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8976    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8977    /// at source rather than assumed, since a citation is a claim about another
8978    /// file and ages like one. Recorded because a harness that structurally
8979    /// cannot reach an arm reports "none" for that arm identically to one that
8980    /// covers it and found nothing.
8981    #[tokio::test]
8982    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8983        let handler = ControlHandler::default();
8984        let frame =
8985            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8986
8987        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8988        // any process holding the connection file. Nothing was proved, so nothing
8989        // may be granted beyond the unattested floor.
8990        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8991        assert_eq!(
8992            stamped,
8993            Principal::Direct,
8994            "a caller that proved nothing must not be stamped as a supervised module"
8995        );
8996
8997        // A claimed module_id with a nonce no supervised child was given is a
8998        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8999        // quietly demoted to Direct, or an impersonation attempt looks identical
9000        // to an ordinary unattested connection.
9001        let forged = handler
9002            .route_open_principal(
9003                &frame,
9004                Some(ConsumerIdentity {
9005                    module_id: "aft".to_string(),
9006                    launch_nonce: "not-a-real-nonce".to_string(),
9007                }),
9008            )
9009            .unwrap();
9010        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9011        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9012    }
9013
9014    /// The test above hands `route_open_principal` an identity it built itself,
9015    /// which proves the stamping rule and nothing about where the identity comes
9016    /// from. The real producer is a wire body, and the two are joined by a serde
9017    /// field name that nothing else asserts.
9018    ///
9019    /// That join fails quietly in one specific way: an unrecognised key is simply
9020    /// absent after parsing, so a renamed or misspelled `consumer_identity`
9021    /// yields `None` and every supervised module silently drops to `Direct`.
9022    /// Capability-wise that is the safe direction, but it surfaces far from its
9023    /// cause — as a module mysteriously refused bash — and it would pass every
9024    /// test that builds its own input.
9025    ///
9026    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
9027    /// would break every client the moment the daemon gains a field, trading a
9028    /// quiet demotion for a hard refusal on additive change. Asserting the join
9029    /// instead means a rename breaks a test here rather than the fleet.
9030    #[test]
9031    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9032        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9033        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9034        let ClientControlRequest::RouteOpen {
9035            consumer_identity, ..
9036        } = parsed
9037        else {
9038            panic!("route.open body must parse as RouteOpen");
9039        };
9040        assert_eq!(
9041            consumer_identity,
9042            Some(ConsumerIdentity {
9043                module_id: "aft".to_string(),
9044                launch_nonce: "n".to_string(),
9045            }),
9046            "the wire field name must reach the value route_open_principal reads"
9047        );
9048    }
9049
9050    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9051        ModuleManifest::builder(module_id, "0.1.0")
9052            .protocol_ver(protocol_ver)
9053            .provides(vec![ProviderRole::ToolProvider {
9054                tools: vec![Tool {
9055                    name: "read".to_string(),
9056                    description: None,
9057                    execution_mode: ExecutionMode::Pure,
9058                    schema: json!({"type": "object"}),
9059                }],
9060                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9061                concurrency: Concurrency::ModuleManaged,
9062                emits_push: true,
9063                sub_supervises: true,
9064            }])
9065            .build()
9066    }
9067
9068    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9069        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9070    }
9071
9072    fn hello_frame_with_control_ops(
9073        module_id: &str,
9074        protocol_ver: u8,
9075        corr: u64,
9076        control_ops: Option<Vec<String>>,
9077    ) -> Frame {
9078        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9079    }
9080
9081    fn hello_frame_with_nonce(
9082        module_id: &str,
9083        protocol_ver: u8,
9084        corr: u64,
9085        launch_nonce: Option<&str>,
9086    ) -> Frame {
9087        hello_frame_full(
9088            module_id,
9089            protocol_ver,
9090            corr,
9091            None,
9092            launch_nonce.map(ToOwned::to_owned),
9093        )
9094    }
9095
9096    fn hello_frame_full(
9097        module_id: &str,
9098        protocol_ver: u8,
9099        corr: u64,
9100        control_ops: Option<Vec<String>>,
9101        launch_nonce: Option<String>,
9102    ) -> Frame {
9103        let body = serde_json::to_vec(&ModuleHelloBody {
9104            manifest: manifest(module_id, protocol_ver),
9105            protocol_ver,
9106            control_ops,
9107            launch_nonce,
9108        })
9109        .unwrap();
9110        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9111    }
9112
9113    fn hello_frame_with_rich_ui(
9114        module_id: &str,
9115        protocol_ver: u8,
9116        corr: u64,
9117        launch_nonce: Option<&str>,
9118        rich_ui: Value,
9119    ) -> Frame {
9120        let mut body = serde_json::to_value(ModuleHelloBody {
9121            manifest: manifest(module_id, protocol_ver),
9122            protocol_ver,
9123            control_ops: None,
9124            launch_nonce: launch_nonce.map(str::to_string),
9125        })
9126        .unwrap();
9127        body["manifest"]["rich_ui"] = rich_ui;
9128        Frame::build(
9129            FrameType::Hello,
9130            control_flags(),
9131            0,
9132            0,
9133            corr,
9134            serde_json::to_vec(&body).unwrap(),
9135        )
9136        .unwrap()
9137    }
9138
9139    fn non_routable_hello_frame_with_control_ops(
9140        module_id: &str,
9141        corr: u64,
9142        control_ops: Option<Vec<String>>,
9143    ) -> Frame {
9144        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9145        manifest.provides.clear();
9146        let body = serde_json::to_vec(&ModuleHelloBody {
9147            manifest,
9148            protocol_ver: PROTOCOL_VERSION,
9149            control_ops,
9150            launch_nonce: None,
9151        })
9152        .unwrap();
9153        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9154    }
9155
9156    fn capability_grammar_hello_frame(
9157        capabilities: Value,
9158        runtime_computed: Option<Value>,
9159        corr: u64,
9160    ) -> Frame {
9161        let mut body = serde_json::to_value(ModuleHelloBody {
9162            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9163            protocol_ver: PROTOCOL_VERSION,
9164            control_ops: None,
9165            launch_nonce: None,
9166        })
9167        .expect("HELLO body serializes");
9168        body["manifest"]["capabilities"] = capabilities;
9169        if let Some(runtime_computed) = runtime_computed {
9170            body["runtime_computed"] = runtime_computed;
9171        }
9172        Frame::build(
9173            FrameType::Hello,
9174            control_flags(),
9175            0,
9176            0,
9177            corr,
9178            serde_json::to_vec(&body).expect("HELLO body reserializes"),
9179        )
9180        .expect("HELLO frame builds")
9181    }
9182
9183    fn channel_request(channel: u16, corr: u64) -> Frame {
9184        Frame::build(
9185            FrameType::Request,
9186            Flags::new(true, Priority::Interactive, false),
9187            channel,
9188            0,
9189            corr,
9190            b"opaque".to_vec(),
9191        )
9192        .unwrap()
9193    }
9194
9195    fn route_ctx(
9196        connection_id: ConnectionId,
9197    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9198        let (tx, rx) = mpsc::channel(8);
9199        (
9200            RouteCtx {
9201                connection_id,
9202                egress: FrameSink::new(tx),
9203            },
9204            rx,
9205        )
9206    }
9207
9208    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9209        serde_json::from_slice(&frame.body).unwrap()
9210    }
9211
9212    /// Register a module over a connection that has a sink and return the
9213    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
9214    /// module's own sink rather than returning it as a reply, so the ack is
9215    /// taken off `rx` here and whatever the test reads next is what followed it.
9216    async fn hello_via_sink(
9217        handler: &ControlHandler,
9218        ctx: &RouteCtx,
9219        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9220        hello: Frame,
9221    ) -> Frame {
9222        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9223        assert!(
9224            replies.is_empty(),
9225            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9226        );
9227        let ack = rx
9228            .try_recv()
9229            .expect("HELLO_ACK is queued on the module sink")
9230            .frame;
9231        assert_eq!(ack.header.ty, FrameType::HelloAck);
9232        ack
9233    }
9234
9235    fn parse_error(frame: &Frame) -> Value {
9236        serde_json::from_slice(&frame.body).unwrap()
9237    }
9238
9239    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9240        serde_json::from_slice(&frame.body).unwrap()
9241    }
9242
9243    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9244        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9245            route_channel,
9246            route_epoch: 0,
9247            kind,
9248        })
9249        .unwrap();
9250        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9251    }
9252
9253    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9254        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9255            module_id: module_id.to_string(),
9256        })
9257        .unwrap();
9258        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9259    }
9260
9261    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9262        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9263    }
9264
9265    // Exercise retired opener fields without putting them back in BindIdentity.
9266    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9267        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9268        let mut frame = route_open_frame(corr, module_id, root);
9269        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9270        body["identity"]["project_root"] = serde_json::json!(path.as_path());
9271        frame.body = serde_json::to_vec(&body).unwrap();
9272        frame
9273    }
9274
9275    fn route_open_frame_with_consumer_capabilities(
9276        corr: u64,
9277        module_id: &str,
9278        _project_root: ScratchDir,
9279        consumer_capabilities: Option<Vec<String>>,
9280    ) -> Frame {
9281        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9282            busy_refusals: None,
9283            target: RouteTarget::ToolProvider {
9284                module_id: module_id.to_string(),
9285            },
9286            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9287            consumer_identity: None,
9288            consumer_capabilities,
9289            role_versions: None,
9290            admission_facts: None,
9291            scope: None,
9292        })
9293        .unwrap();
9294        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9295    }
9296
9297    fn route_open_frame_with_role_versions(
9298        corr: u64,
9299        module_id: &str,
9300        _project_root: ScratchDir,
9301        role_versions: Option<BTreeMap<String, String>>,
9302    ) -> Frame {
9303        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9304            busy_refusals: None,
9305            target: RouteTarget::ToolProvider {
9306                module_id: module_id.to_string(),
9307            },
9308            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9309            consumer_identity: None,
9310            consumer_capabilities: None,
9311            role_versions,
9312            admission_facts: None,
9313            scope: None,
9314        })
9315        .unwrap();
9316        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9317    }
9318
9319    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9320        entries
9321            .iter()
9322            .map(|(role, version)| (role.to_string(), version.to_string()))
9323            .collect()
9324    }
9325
9326    fn route_open_frame_with_admission_facts(
9327        corr: u64,
9328        module_id: &str,
9329        _project_root: ScratchDir,
9330        consumer_identity: Option<subc_control::ConsumerIdentity>,
9331        facts: Option<Value>,
9332    ) -> Frame {
9333        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9334            busy_refusals: None,
9335            target: RouteTarget::ToolProvider {
9336                module_id: module_id.to_string(),
9337            },
9338            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9339            consumer_identity,
9340            consumer_capabilities: None,
9341            role_versions: None,
9342            admission_facts: facts,
9343            scope: None,
9344        })
9345        .unwrap();
9346        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9347    }
9348
9349    #[derive(Clone, Default)]
9350    struct EventCapture {
9351        events: Arc<Mutex<Vec<CapturedEvent>>>,
9352    }
9353
9354    #[derive(Clone, Debug)]
9355    struct CapturedEvent {
9356        target: String,
9357        level: tracing::Level,
9358        fields: BTreeMap<String, String>,
9359    }
9360
9361    impl EventCapture {
9362        fn events(&self) -> Vec<CapturedEvent> {
9363            self.events.lock().unwrap().clone()
9364        }
9365    }
9366
9367    impl<S> Layer<S> for EventCapture
9368    where
9369        S: Subscriber,
9370    {
9371        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9372            let mut visitor = EventFieldVisitor::default();
9373            event.record(&mut visitor);
9374            self.events.lock().unwrap().push(CapturedEvent {
9375                target: event.metadata().target().to_string(),
9376                level: *event.metadata().level(),
9377                fields: visitor.fields,
9378            });
9379        }
9380    }
9381
9382    #[derive(Default)]
9383    struct EventFieldVisitor {
9384        fields: BTreeMap<String, String>,
9385    }
9386
9387    impl Visit for EventFieldVisitor {
9388        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9389            self.fields
9390                .insert(field.name().to_string(), format!("{value:?}"));
9391        }
9392    }
9393
9394    fn health_response(corr: u64, status: HealthStatus) -> Frame {
9395        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9396            status,
9397            detail: Some("warming".to_string()),
9398            metrics: Some(json!({"queue_depth": 3})),
9399        })
9400        .unwrap();
9401        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9402    }
9403
9404    fn route_bind_ack(corr: u64) -> Frame {
9405        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9406        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9407    }
9408
9409    fn unique_project_root(label: &str) -> ScratchDir {
9410        ScratchDir::new(label)
9411    }
9412
9413    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9414        match parse_route_poll(frame) {
9415            ClientControlResponse::RoutePoll {
9416                status: None,
9417                live: Some(live),
9418                ..
9419            } => assert_eq!(live, expected_live),
9420            other => panic!("unexpected route.poll response: {other:?}"),
9421        }
9422    }
9423
9424    fn bind_liveness_route(
9425        registry: &Registry,
9426        forwarding: &ForwardingTable,
9427        module_id: &str,
9428    ) -> (RouteCtx, u16, u32) {
9429        let module_connection = ConnectionId::new(101);
9430        let client_connection = ConnectionId::new(202);
9431        let registration = registry
9432            .register_with_control_ops(
9433                manifest(module_id, PROTOCOL_VERSION),
9434                PROTOCOL_VERSION,
9435                module_connection,
9436                module_baseline_control_ops(),
9437            )
9438            .unwrap();
9439        let (module_tx, _module_rx) = mpsc::channel(8);
9440        let endpoint = forwarding
9441            .register_module_connection(
9442                module_connection,
9443                module_id.to_string(),
9444                PROTOCOL_VERSION,
9445                manifest_concurrency(&registration.manifest),
9446                FrameSink::new(module_tx),
9447            )
9448            .unwrap();
9449        let (client_ctx, _client_rx) = route_ctx(client_connection);
9450        let pending = forwarding
9451            .begin_route_bind_relay_for_test(
9452                client_connection,
9453                client_ctx.egress.clone(),
9454                1,
9455                module_id,
9456            )
9457            .unwrap();
9458        assert_eq!(pending.endpoint, endpoint);
9459        let route_channel = pending.client_channel;
9460        let route_epoch = pending.client_epoch;
9461        forwarding
9462            .complete_pending_relay(
9463                module_connection,
9464                pending.corr,
9465                RouteBindRelayOutcome::Accepted,
9466            )
9467            .unwrap();
9468        (client_ctx, route_channel, route_epoch)
9469    }
9470
9471    struct FakeProcessLiveness {
9472        live: Option<bool>,
9473    }
9474
9475    impl ModuleProcessLiveness for FakeProcessLiveness {
9476        fn process_live(&self, _module_id: &str) -> Option<bool> {
9477            self.live
9478        }
9479    }
9480
9481    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9482    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9483    {
9484        let registry = Arc::new(Registry::default());
9485        let supervisor_handle = SupervisorHandle::new();
9486        let supervisor = Supervisor::new_for_test(
9487            Arc::clone(&registry),
9488            RestartPolicy::new(1, Duration::from_millis(10)),
9489        )
9490        .with_handle(supervisor_handle.clone());
9491        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9492        let module = supervisor
9493            .spawn(ModuleSpec {
9494                module_id: "stderr-tail-wire".to_string(),
9495                program: fake_aft_stub_path(),
9496                args: Vec::new(),
9497                env: vec![
9498                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9499                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9500                ],
9501                reserved: false,
9502                reserved_prefixes: Vec::new(),
9503                protocol: ModuleProtocol::Subc,
9504                overlap: Default::default(),
9505            })
9506            .unwrap();
9507
9508        let deadline = Instant::now() + Duration::from_secs(5);
9509        loop {
9510            let tail = module.stderr_tail(None, None);
9511            if tail
9512                .entries
9513                .iter()
9514                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9515                && tail.entries.iter().any(|entry| {
9516                    matches!(
9517                        entry,
9518                        TailEntry::Line {
9519                            truncated: true,
9520                            ..
9521                        }
9522                    )
9523                })
9524            {
9525                break;
9526            }
9527            assert!(
9528                Instant::now() < deadline,
9529                "module did not produce a truncated line and restart boundary: {tail:?}"
9530            );
9531            sleep(Duration::from_millis(10)).await;
9532        }
9533
9534        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9535        let request = ClientControlRequest::SupervisorStderrTail {
9536            module_id: "stderr-tail-wire".to_string(),
9537            max_lines: None,
9538            max_bytes: None,
9539        };
9540        let frame = Frame::build(
9541            FrameType::Request,
9542            control_flags(),
9543            0,
9544            0,
9545            1,
9546            serde_json::to_vec(&request).unwrap(),
9547        )
9548        .unwrap();
9549        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9550        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9551        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9552            serde_json::from_slice(&responses[0].body).unwrap()
9553        else {
9554            panic!("expected supervisor.stderr_tail response");
9555        };
9556
9557        assert!(
9558            tail.entries
9559                .iter()
9560                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9561            "the control response lost the restart boundary"
9562        );
9563        let Some(StderrTailEntry::Line {
9564            text,
9565            truncated,
9566            at_ms,
9567        }) = tail.entries.iter().find(|entry| {
9568            matches!(
9569                entry,
9570                StderrTailEntry::Line {
9571                    truncated: true,
9572                    ..
9573                }
9574            )
9575        })
9576        else {
9577            panic!("the control response lost the truncated line");
9578        };
9579        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9580        assert!(*truncated);
9581        assert!(
9582            at_ms.is_some(),
9583            "the control response lost the line's capture time"
9584        );
9585    }
9586
9587    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9588    /// read done on the worker thread would stall every other task until it
9589    /// finished; the read must run off the worker so this test's own task keeps
9590    /// running while the read is paused.
9591    #[tokio::test(flavor = "current_thread")]
9592    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9593        let dir = ScratchDir::new("terminals-off-worker");
9594        let journal_path = dir.join("terminals.jsonl");
9595        let registry = Arc::new(Registry::default());
9596        let supervisor_handle = SupervisorHandle::new();
9597        let supervisor =
9598            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9599                .with_handle(supervisor_handle.clone())
9600                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9601        let module = supervisor
9602            .spawn(ModuleSpec {
9603                module_id: "terminal-off-worker".to_string(),
9604                program: fake_aft_stub_path(),
9605                args: Vec::new(),
9606                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9607                reserved: false,
9608                reserved_prefixes: Vec::new(),
9609                protocol: ModuleProtocol::Subc,
9610                overlap: Default::default(),
9611            })
9612            .unwrap();
9613        let deadline = Instant::now() + Duration::from_secs(5);
9614        while module.terminal_history().entries.len() != 2 {
9615            assert!(Instant::now() < deadline, "module did not record two exits");
9616            sleep(Duration::from_millis(10)).await;
9617        }
9618
9619        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9620        let handler =
9621            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9622        let frame = Frame::build(
9623            FrameType::Request,
9624            control_flags(),
9625            0,
9626            0,
9627            1,
9628            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9629                module_id: "terminal-off-worker".to_string(),
9630            })
9631            .unwrap(),
9632        )
9633        .unwrap();
9634        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9635        let spawned_at = std::time::Instant::now();
9636        let read = tokio::spawn({
9637            let handler = Arc::clone(&handler);
9638            async move { handler.handle_control_frame(&ctx, frame).await }
9639        });
9640        // Waiting for the pause from a blocking thread keeps this task pending,
9641        // so the runtime's single worker is free to run the read task.
9642        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9643            .await
9644            .unwrap()
9645            .expect("the history read reached its pause");
9646        let elapsed = spawned_at.elapsed();
9647        assert!(
9648            elapsed < Duration::from_secs(2) && !read.is_finished(),
9649            "this task could not run while the history read was paused \
9650             (resumed after {elapsed:?}, read finished: {})",
9651            read.is_finished()
9652        );
9653
9654        drop(release);
9655        let responses = read.await.unwrap().unwrap();
9656        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9657        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9658            panic!("expected supervisor.terminals response");
9659        };
9660        assert_eq!(terminals.entries.len(), 2);
9661        assert_eq!(terminals.journal_skipped_lines, 0);
9662        assert_eq!(terminals.journal_read_errors, 0);
9663    }
9664
9665    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9666    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9667        let registry = Arc::new(Registry::default());
9668        let supervisor_handle = SupervisorHandle::new();
9669        let supervisor =
9670            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9671                .with_handle(supervisor_handle.clone());
9672        let module = supervisor
9673            .spawn(ModuleSpec {
9674                module_id: "terminal-golden".to_string(),
9675                program: fake_aft_stub_path(),
9676                args: Vec::new(),
9677                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9678                reserved: false,
9679                reserved_prefixes: Vec::new(),
9680                protocol: ModuleProtocol::Subc,
9681                overlap: Default::default(),
9682            })
9683            .unwrap();
9684
9685        let deadline = Instant::now() + Duration::from_secs(5);
9686        while module.terminal_history().entries.len() != 2 {
9687            assert!(
9688                Instant::now() < deadline,
9689                "module did not retain two terminal exits: {:?}",
9690                module.terminal_history()
9691            );
9692            sleep(Duration::from_millis(10)).await;
9693        }
9694
9695        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9696        let request = ClientControlRequest::SupervisorTerminals {
9697            module_id: "terminal-golden".to_string(),
9698        };
9699        let frame = Frame::build(
9700            FrameType::Request,
9701            control_flags(),
9702            0,
9703            0,
9704            1,
9705            serde_json::to_vec(&request).unwrap(),
9706        )
9707        .unwrap();
9708        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9709        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9710        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9711        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9712            panic!("expected supervisor.terminals response");
9713        };
9714        assert_eq!(terminals.entries.len(), 2);
9715        assert_eq!(terminals.dropped, 0);
9716
9717        let mut rendered = serde_json::to_value(response).unwrap();
9718        // Wall-clock fields are the observation contract, but not stable fixture
9719        // bytes; normalize only them after the real handler has shaped the response.
9720        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9721        for (index, entry) in rendered["entries"]
9722            .as_array_mut()
9723            .expect("terminal response entries array")
9724            .iter_mut()
9725            .enumerate()
9726        {
9727            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9728        }
9729
9730        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9731            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9732        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9733        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9734            std::fs::write(&golden_path, &serialized).unwrap();
9735        }
9736        let expected: Value =
9737            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9738        assert_eq!(rendered, expected);
9739    }
9740
9741    #[test]
9742    fn hello_registers_manifest_and_returns_ack() {
9743        let registry = Arc::new(Registry::default());
9744        let handler = ControlHandler::new(Arc::clone(&registry));
9745        let conn = ConnectionId::new(1);
9746
9747        let responses = handler
9748            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9749            .unwrap();
9750
9751        assert_eq!(responses.len(), 1);
9752        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9753        assert_eq!(responses[0].header.channel, 0);
9754        assert_eq!(responses[0].header.corr, 7);
9755        let ack = parse_ack(&responses[0]);
9756        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9757        assert!(ack
9758            .subc_capabilities
9759            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9760        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9761        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9762        assert!(ack
9763            .subc_ops
9764            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9765        assert!(ack
9766            .subc_ops
9767            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9768
9769        let registration = registry.get_module("aft").unwrap().unwrap();
9770        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9771        assert_eq!(registration.state, ChannelState::Active);
9772        assert_eq!(registration.connection_id, conn);
9773        assert_eq!(registration.control_ops, module_baseline_control_ops());
9774    }
9775
9776    #[test]
9777    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9778        let invalid_identifiers = [
9779            ("case_change", "credentials-Provider/v1"),
9780            ("leading_zero", "credentials-provider/v01"),
9781            ("trailing_hyphen", "credentials-provider-/v1"),
9782            ("consecutive_hyphens", "credentials--provider/v1"),
9783            ("uppercase", "Credentials-provider/v1"),
9784            ("missing_v", "credentials-provider/1"),
9785            ("whitespace", "credentials provider/v1"),
9786            ("zero_version", "credentials-provider/v0"),
9787            ("out_of_range_version", "credentials-provider/v4294967296"),
9788            (
9789                "overlength_name",
9790                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9791            ),
9792        ];
9793        let mut cases = invalid_identifiers
9794            .into_iter()
9795            .map(|(name, identifier)| {
9796                (
9797                    format!("identifier_{name}"),
9798                    "capabilities.provides[0]".to_string(),
9799                    identifier.to_string(),
9800                    json!({ "provides": [identifier] }),
9801                    None,
9802                )
9803            })
9804            .collect::<Vec<_>>();
9805        cases.extend([
9806            (
9807                "unknown_need".to_string(),
9808                "capabilities.requires[0].need".to_string(),
9809                "deferred".to_string(),
9810                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9811                None,
9812            ),
9813            (
9814                "duplicate_provides".to_string(),
9815                "capabilities.provides[1]".to_string(),
9816                "credentials-provider/v1".to_string(),
9817                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9818                None,
9819            ),
9820            (
9821                "duplicate_must_never_reach".to_string(),
9822                "capabilities.must_never_reach[1]".to_string(),
9823                "credentials-provider/v1".to_string(),
9824                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9825                None,
9826            ),
9827            (
9828                "duplicate_requires_same_need".to_string(),
9829                "capabilities.requires[1]".to_string(),
9830                "credentials-provider/v1".to_string(),
9831                json!({ "requires": [
9832                    { "capability": "credentials-provider/v1", "need": "required" },
9833                    { "capability": "credentials-provider/v1", "need": "required" }
9834                ] }),
9835                None,
9836            ),
9837            (
9838                "duplicate_requires_conflicting_need".to_string(),
9839                "capabilities.requires[1]".to_string(),
9840                "credentials-provider/v1".to_string(),
9841                json!({ "requires": [
9842                    { "capability": "credentials-provider/v1", "need": "required" },
9843                    { "capability": "credentials-provider/v1", "need": "optional" }
9844                ] }),
9845                None,
9846            ),
9847            (
9848                "capabilities_root_pointer".to_string(),
9849                "runtime_computed[0]".to_string(),
9850                "/capabilities".to_string(),
9851                json!({}),
9852                Some(json!(["/capabilities"])),
9853            ),
9854            (
9855                "capabilities_descendant_pointer".to_string(),
9856                "runtime_computed[0]".to_string(),
9857                "/capabilities/provides".to_string(),
9858                json!({}),
9859                Some(json!(["/capabilities/provides"])),
9860            ),
9861            (
9862                "malformed_pointer_without_leading_slash".to_string(),
9863                "runtime_computed[0]".to_string(),
9864                "capabilities".to_string(),
9865                json!({}),
9866                Some(json!(["capabilities"])),
9867            ),
9868            (
9869                "malformed_pointer_escape".to_string(),
9870                "runtime_computed[0]".to_string(),
9871                "/roles/~2/tools".to_string(),
9872                json!({}),
9873                Some(json!(["/roles/~2/tools"])),
9874            ),
9875            (
9876                "unknown_capabilities_field".to_string(),
9877                "capabilities.future".to_string(),
9878                "<array>".to_string(),
9879                json!({ "future": [] }),
9880                None,
9881            ),
9882        ]);
9883
9884        for (index, (name, field, value, capabilities, runtime_computed)) in
9885            cases.into_iter().enumerate()
9886        {
9887            let registry = Arc::new(Registry::default());
9888            let handler = ControlHandler::new(Arc::clone(&registry));
9889            let response = handler
9890                .handle_control(
9891                    ConnectionId::new((index + 1) as u64),
9892                    capability_grammar_hello_frame(
9893                        capabilities,
9894                        runtime_computed,
9895                        index as u64 + 1,
9896                    ),
9897                )
9898                .expect("invalid HELLO returns a refusal");
9899
9900            assert_eq!(response.len(), 1, "{name} must emit one refusal");
9901            let error = parse_error(&response[0]);
9902            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9903            let message = error["message"]
9904                .as_str()
9905                .expect("error message is a string");
9906            assert!(
9907                message.contains(&field),
9908                "{name}: field missing from {message}"
9909            );
9910            assert!(
9911                message.contains(&value),
9912                "{name}: value missing from {message}"
9913            );
9914            assert_eq!(
9915                registry
9916                    .active_registration_count()
9917                    .expect("registry reads"),
9918                0,
9919                "{name}: refused HELLO must not create a catalog entry"
9920            );
9921        }
9922    }
9923
9924    #[test]
9925    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9926        let registry = Arc::new(Registry::default());
9927        let handler = ControlHandler::new(Arc::clone(&registry));
9928        let capabilities = json!({
9929            "provides": ["credentials-provider/v1"],
9930            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9931            "must_never_reach": ["federation-transport/v1"]
9932        });
9933        let response = handler
9934            .handle_control(
9935                ConnectionId::new(99),
9936                capability_grammar_hello_frame(
9937                    capabilities.clone(),
9938                    Some(json!(["/roles/0/tools"])),
9939                    99,
9940                ),
9941            )
9942            .expect("valid HELLO registers");
9943        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9944
9945        let request = Frame::build(
9946            FrameType::Request,
9947            control_flags(),
9948            0,
9949            0,
9950            100,
9951            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9952                .expect("catalog request serializes"),
9953        )
9954        .expect("catalog request frame builds");
9955        let response = handler
9956            .handle_catalog_list(request, None)
9957            .expect("catalog list succeeds");
9958        let ClientControlResponse::CatalogList { modules, .. } =
9959            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9960        else {
9961            panic!("catalog request must return catalog.list");
9962        };
9963        assert_eq!(modules.len(), 1);
9964        assert_eq!(
9965            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9966            capabilities
9967        );
9968    }
9969
9970    #[test]
9971    fn catalog_list_mirrors_management_operation_description() {
9972        let registry = Arc::new(Registry::default());
9973        let handler = ControlHandler::new(Arc::clone(&registry));
9974        let description = "List managed records and return their identifiers and metadata.";
9975        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9976        manifest.provides = vec![ProviderRole::ManagementSurface {
9977            operations: vec![ManagementOperation {
9978                name: "records.list".to_string(),
9979                kind: ManagementOperationKind::Query,
9980                description: Some(description.to_string()),
9981            }],
9982            config_schema: json!({"type": "object"}),
9983            observability: vec![ObservabilitySurface {
9984                name: "records.stats".to_string(),
9985                kind: ObservabilityKind::Snapshot,
9986            }],
9987            identity_scope: vec![IdentityScope::Project],
9988            concurrency: Concurrency::ModuleManaged,
9989        }];
9990        registry
9991            .register_with_control_ops(
9992                manifest,
9993                PROTOCOL_VERSION,
9994                ConnectionId::new(99),
9995                Vec::new(),
9996            )
9997            .expect("described management manifest registers");
9998
9999        let request = Frame::build(
10000            FrameType::Request,
10001            control_flags(),
10002            0,
10003            0,
10004            100,
10005            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10006                .expect("catalog request serializes"),
10007        )
10008        .expect("catalog request frame builds");
10009        let response = handler
10010            .handle_catalog_list(request, None)
10011            .expect("catalog list succeeds");
10012        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10013        assert_eq!(
10014            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10015            "catalog.list must preserve the declared operation description verbatim"
10016        );
10017    }
10018
10019    #[test]
10020    fn catalog_list_mirrors_registered_operation_delivery() {
10021        let registry = Arc::new(Registry::default());
10022        let handler = ControlHandler::new(Arc::clone(&registry));
10023        let delivery =
10024            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10025        let mut body = serde_json::to_value(ModuleHelloBody {
10026            manifest: manifest("watch-management", PROTOCOL_VERSION),
10027            protocol_ver: PROTOCOL_VERSION,
10028            control_ops: None,
10029            launch_nonce: None,
10030        })
10031        .unwrap();
10032        body["manifest"]["provides"] = json!([{
10033            "role": "management_surface",
10034            "operations": [
10035                {"name": "status", "kind": "query"},
10036                {"name": "future", "kind": "query"},
10037                {"name": "reset", "kind": "mutate"}
10038            ],
10039            "config_schema": {}, "observability": [], "identity_scope": []
10040        }]);
10041        body["manifest"]["operation_delivery"] = delivery.clone();
10042        let hello = Frame::build(
10043            FrameType::Hello,
10044            control_flags(),
10045            0,
10046            0,
10047            99,
10048            serde_json::to_vec(&body).unwrap(),
10049        )
10050        .unwrap();
10051        let response = handler
10052            .handle_control(ConnectionId::new(99), hello)
10053            .unwrap();
10054        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10055
10056        let (_, registrations) = registry.list_modules().unwrap();
10057        let registered = &registrations[0].manifest;
10058        assert!(registered.is_watch_operation("status"));
10059        assert!(!registered.is_watch_operation("future"));
10060        assert!(!registered.is_watch_operation("reset"));
10061        assert!(!registered.is_watch_operation("missing"));
10062
10063        let request = Frame::build(
10064            FrameType::Request,
10065            control_flags(),
10066            0,
10067            0,
10068            100,
10069            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10070        )
10071        .unwrap();
10072        let response = handler.handle_catalog_list(request, None).unwrap();
10073        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10074        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10075        let ClientControlResponse::CatalogList { modules, .. } =
10076            serde_json::from_slice(&response[0].body).unwrap()
10077        else {
10078            panic!("expected catalog.list");
10079        };
10080        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10081    }
10082
10083    #[test]
10084    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10085        let registry = Arc::new(Registry::default());
10086        let supervisor = SupervisorHandle::new();
10087        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10088        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10089        let descriptor = json!({
10090            "schema": "ckios.richui.manifest/1",
10091            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10092            "bytes": 4096,
10093            "chunks": 4
10094        });
10095        let connection = ConnectionId::new(201);
10096        let hello = handler
10097            .handle_control(
10098                connection,
10099                hello_frame_with_rich_ui(
10100                    "vault",
10101                    PROTOCOL_VERSION,
10102                    201,
10103                    Some("the-real-nonce"),
10104                    descriptor.clone(),
10105                ),
10106            )
10107            .unwrap();
10108        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10109
10110        let request =
10111            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10112        let provides = registry
10113            .get_module_by_connection(connection)
10114            .unwrap()
10115            .unwrap()
10116            .manifest
10117            .provides;
10118        handler
10119            .handle_catalog_update(connection, request, provides, None, Some(false))
10120            .unwrap();
10121
10122        let request = Frame::build(
10123            FrameType::Request,
10124            control_flags(),
10125            0,
10126            0,
10127            203,
10128            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10129        )
10130        .unwrap();
10131        let response = handler.handle_catalog_list(request, None).unwrap();
10132        let ClientControlResponse::CatalogList { modules, .. } =
10133            serde_json::from_slice(&response[0].body).unwrap()
10134        else {
10135            panic!("expected catalog.list");
10136        };
10137        assert_eq!(
10138            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10139            descriptor
10140        );
10141        assert!(modules[0].nonce_verified);
10142    }
10143
10144    fn hello_with_role_declarations(declarations: Value) -> Frame {
10145        let mut body = serde_json::to_value(ModuleHelloBody {
10146            manifest: manifest("role-test", PROTOCOL_VERSION),
10147            protocol_ver: PROTOCOL_VERSION,
10148            control_ops: None,
10149            launch_nonce: None,
10150        })
10151        .unwrap();
10152        body["manifest"]["role_declarations"] = declarations;
10153        Frame::build(
10154            FrameType::Hello,
10155            control_flags(),
10156            0,
10157            0,
10158            301,
10159            serde_json::to_vec(&body).unwrap(),
10160        )
10161        .unwrap()
10162    }
10163
10164    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10165        let request = Frame::build(
10166            FrameType::Request,
10167            control_flags(),
10168            0,
10169            0,
10170            302,
10171            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10172        )
10173        .unwrap();
10174        let response = handler.handle_catalog_list(request, None).unwrap();
10175        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10176        let ClientControlResponse::CatalogList { mut modules, .. } =
10177            serde_json::from_slice(&response[0].body).unwrap()
10178        else {
10179            panic!("expected catalog.list");
10180        };
10181        (raw["modules"][0].clone(), modules.remove(0))
10182    }
10183
10184    #[test]
10185    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10186        let registry = Arc::new(Registry::default());
10187        let handler = ControlHandler::new(Arc::clone(&registry));
10188        let declarations = json!({
10189            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10190            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10191        });
10192        let connection = ConnectionId::new(301);
10193        let response = handler
10194            .handle_control(
10195                connection,
10196                hello_with_role_declarations(declarations.clone()),
10197            )
10198            .unwrap();
10199        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10200        let registered = registry
10201            .get_module_by_connection(connection)
10202            .unwrap()
10203            .unwrap()
10204            .manifest;
10205        // Compared as JSON values, not bytes: object key order is not part of
10206        // the relay's promise, and workspace builds unify serde_json's
10207        // preserve_order feature, which changes the order a literal keeps.
10208        assert_eq!(
10209            serde_json::to_value(&registered.role_declarations).unwrap(),
10210            declarations
10211        );
10212        let (raw, entry) = role_declarations_catalog(&handler);
10213        assert_eq!(raw["role_declarations"], declarations);
10214        assert_eq!(entry.role_declarations, registered.role_declarations);
10215
10216        let request =
10217            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10218        handler
10219            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10220            .unwrap();
10221        let (raw, entry) = role_declarations_catalog(&handler);
10222        assert_eq!(raw["role_declarations"], declarations);
10223        assert_eq!(entry.role_declarations, registered.role_declarations);
10224    }
10225
10226    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10227        let registry = Arc::new(Registry::default());
10228        let handler = ControlHandler::new(Arc::clone(&registry));
10229        let capture = EventCapture::default();
10230        let _subscriber =
10231            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10232        let response = handler
10233            .handle_control(
10234                ConnectionId::new(301),
10235                hello_with_role_declarations(declarations),
10236            )
10237            .unwrap();
10238        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10239        assert!(registry
10240            .get_module_by_connection(ConnectionId::new(301))
10241            .unwrap()
10242            .unwrap()
10243            .manifest
10244            .role_declarations
10245            .is_empty());
10246        let (raw, entry) = role_declarations_catalog(&handler);
10247        assert!(raw.get("role_declarations").is_none());
10248        assert!(entry.role_declarations.is_empty());
10249        let warnings: Vec<_> = capture
10250            .events()
10251            .into_iter()
10252            .filter(|event| {
10253                event.fields.get("message").map(String::as_str)
10254                    == Some("dropping whole role_declarations map from HELLO")
10255            })
10256            .collect();
10257        assert_eq!(warnings.len(), 1);
10258        assert_eq!(warnings[0].level, tracing::Level::WARN);
10259        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10260        assert_eq!(warnings[0].fields["reason"], reason);
10261    }
10262
10263    #[test]
10264    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10265        for value in [
10266            json!(null),
10267            json!(false),
10268            json!(0),
10269            json!("opaque"),
10270            json!([]),
10271        ] {
10272            assert_hello_drops_role_declarations_with_warning(
10273                json!({"routing/v1": {}, "broken/v1": value}),
10274                "role_declarations values must be objects",
10275            );
10276        }
10277        assert_hello_drops_role_declarations_with_warning(
10278            json!([]),
10279            "role_declarations must be an object",
10280        );
10281    }
10282
10283    #[test]
10284    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10285        assert_hello_drops_role_declarations_with_warning(
10286            json!({"routing/v1": {}, "bad key": {}}),
10287            "role_declarations key must use capability-name grammar",
10288        );
10289    }
10290
10291    #[test]
10292    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10293        assert_hello_drops_role_declarations_with_warning(
10294            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10295            "role_declarations exceeds 16384 compact JSON bytes",
10296        );
10297    }
10298
10299    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10300        let mut body = serde_json::to_value(ModuleHelloBody {
10301            manifest: manifest("flow-ops", PROTOCOL_VERSION),
10302            protocol_ver: PROTOCOL_VERSION,
10303            control_ops: None,
10304            launch_nonce: None,
10305        })
10306        .unwrap();
10307        body["manifest"]["provides"] = json!([{
10308            "role": "management_surface",
10309            "operations": [
10310                {"name": "diagnostics.findings", "kind": "query"},
10311                {"name": "reset", "kind": "mutate"}
10312            ],
10313            "config_schema": {}, "observability": [], "identity_scope": []
10314        }]);
10315        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10316        if let Some(flow_operations) = flow_operations {
10317            body["manifest"]["flow_operations"] = flow_operations;
10318        }
10319        Frame::build(
10320            FrameType::Hello,
10321            control_flags(),
10322            0,
10323            0,
10324            401,
10325            serde_json::to_vec(&body).unwrap(),
10326        )
10327        .unwrap()
10328    }
10329
10330    #[test]
10331    fn catalog_list_mirrors_flow_operations_unchanged() {
10332        // Order is kept as declared (not sorted) and an empty list stays an
10333        // affirmative "none", distinct from an absent declaration.
10334        for (capability, flow_operations) in [
10335            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10336            ("call-modes/v1", json!(["diagnostics.findings"])),
10337            ("flow-scopes/v1", json!([])),
10338        ] {
10339            let registry = Arc::new(Registry::default());
10340            let handler = ControlHandler::new(Arc::clone(&registry));
10341            let connection = ConnectionId::new(401);
10342            let response = handler
10343                .handle_control(
10344                    connection,
10345                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10346                )
10347                .unwrap();
10348            assert_eq!(response[0].header.ty, FrameType::HelloAck);
10349            let registered = registry
10350                .get_module_by_connection(connection)
10351                .unwrap()
10352                .unwrap()
10353                .manifest;
10354            assert_eq!(
10355                serde_json::to_value(&registered.flow_operations).unwrap(),
10356                flow_operations
10357            );
10358            let (raw, entry) = role_declarations_catalog(&handler);
10359            assert_eq!(raw["flow_operations"], flow_operations);
10360            assert_eq!(entry.flow_operations, registered.flow_operations);
10361        }
10362
10363        let registry = Arc::new(Registry::default());
10364        let handler = ControlHandler::new(Arc::clone(&registry));
10365        let response = handler
10366            .handle_control(
10367                ConnectionId::new(401),
10368                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10369            )
10370            .unwrap();
10371        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10372        let (raw, entry) = role_declarations_catalog(&handler);
10373        assert!(raw.get("flow_operations").is_none());
10374        assert_eq!(entry.flow_operations, None);
10375    }
10376
10377    fn assert_hello_drops_flow_operations_with_warning(
10378        capabilities: Value,
10379        flow_operations: Value,
10380        reason: &str,
10381    ) {
10382        let registry = Arc::new(Registry::default());
10383        let handler = ControlHandler::new(Arc::clone(&registry));
10384        let capture = EventCapture::default();
10385        let _subscriber =
10386            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10387        let response = handler
10388            .handle_control(
10389                ConnectionId::new(401),
10390                hello_with_flow_operations(capabilities, Some(flow_operations)),
10391            )
10392            .unwrap();
10393        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10394        let registered = registry
10395            .get_module_by_connection(ConnectionId::new(401))
10396            .unwrap()
10397            .expect("the module still registers")
10398            .manifest;
10399        assert_eq!(registered.module_id, "flow-ops");
10400        assert_eq!(registered.flow_operations, None);
10401        let (raw, entry) = role_declarations_catalog(&handler);
10402        assert_eq!(raw["module_id"], "flow-ops");
10403        assert!(raw.get("flow_operations").is_none());
10404        assert_eq!(entry.flow_operations, None);
10405        let warnings: Vec<_> = capture
10406            .events()
10407            .into_iter()
10408            .filter(|event| {
10409                event.fields.get("message").map(String::as_str)
10410                    == Some("dropping flow_operations from HELLO")
10411            })
10412            .collect();
10413        assert_eq!(warnings.len(), 1);
10414        assert_eq!(warnings[0].level, tracing::Level::WARN);
10415        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10416        assert_eq!(warnings[0].fields["reason"], reason);
10417    }
10418
10419    #[test]
10420    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10421        assert_hello_drops_flow_operations_with_warning(
10422            json!(["flow-scopes/v1"]),
10423            json!(["diagnostics.findings", "missing"]),
10424            "flow_operations names 'missing', which is not a declared management operation",
10425        );
10426    }
10427
10428    #[test]
10429    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10430        assert_hello_drops_flow_operations_with_warning(
10431            json!([]),
10432            json!(["diagnostics.findings"]),
10433            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10434        );
10435        // A run-only capability does not admit flows either.
10436        assert_hello_drops_flow_operations_with_warning(
10437            json!(["agent-run-scopes/v1"]),
10438            json!([]),
10439            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10440        );
10441    }
10442
10443    #[test]
10444    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10445        assert_hello_drops_flow_operations_with_warning(
10446            json!(["flow-scopes/v1"]),
10447            json!(["reset", "reset"]),
10448            "flow_operations names 'reset' more than once",
10449        );
10450        assert_hello_drops_flow_operations_with_warning(
10451            json!(["flow-scopes/v1"]),
10452            json!(vec!["reset"; 257]),
10453            "flow_operations lists 257 names, more than 256",
10454        );
10455        assert_hello_drops_flow_operations_with_warning(
10456            json!(["flow-scopes/v1"]),
10457            json!("reset"),
10458            "flow_operations must be an array of operation names",
10459        );
10460    }
10461
10462    #[test]
10463    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10464        let registry = Arc::new(Registry::default());
10465        let handler = ControlHandler::new(registry.clone());
10466        let descriptor = json!({
10467            "schema": "ckios.richui.manifest/1",
10468            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10469            "bytes": 128,
10470            "chunks": 2
10471        });
10472        let malformed = [
10473            ("schema", json!("")),
10474            ("schema", json!("a".repeat(65))),
10475            ("schema", json!("Bad schema")),
10476            (
10477                "manifest_digest",
10478                json!(format!("sha256:{}", "A".repeat(64))),
10479            ),
10480            ("manifest_digest", json!("sha256:abc")),
10481            ("bytes", json!(0)),
10482            ("bytes", json!(16_777_217)),
10483            ("bytes", json!("128")),
10484            ("chunks", json!(0)),
10485            ("chunks", json!(1025)),
10486            ("chunks", json!("2")),
10487        ];
10488        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10489            let mut invalid = descriptor.clone();
10490            invalid[field] = invalid_value;
10491            let response = handler
10492                .handle_control(
10493                    ConnectionId::new(210 + index as u64),
10494                    hello_frame_with_rich_ui(
10495                        &format!("bad-rich-ui-{index}"),
10496                        PROTOCOL_VERSION,
10497                        210 + index as u64,
10498                        None,
10499                        invalid,
10500                    ),
10501                )
10502                .unwrap();
10503            assert_eq!(response[0].header.ty, FrameType::Error);
10504            let error = parse_error(&response[0]);
10505            assert_eq!(error["code"], "invalid_manifest");
10506            assert!(error["message"]
10507                .as_str()
10508                .unwrap()
10509                .contains(&format!("rich_ui.{field}")));
10510            assert!(registry
10511                .get_module(&format!("bad-rich-ui-{index}"))
10512                .unwrap()
10513                .is_none());
10514        }
10515    }
10516
10517    #[test]
10518    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10519        let registry = Arc::new(Registry::default());
10520        let handler = ControlHandler::new(Arc::clone(&registry));
10521        let connection = ConnectionId::new(219);
10522        registry
10523            .register_with_control_ops(
10524                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10525                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10526                        "bad schema",
10527                        format!("sha256:{}", "a".repeat(64)),
10528                        128,
10529                        2,
10530                    )))
10531                    .build(),
10532                PROTOCOL_VERSION,
10533                connection,
10534                Vec::new(),
10535            )
10536            .unwrap();
10537        let request =
10538            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10539        let response = handler
10540            .handle_catalog_update(connection, request, Vec::new(), None, None)
10541            .unwrap();
10542        assert_eq!(response[0].header.ty, FrameType::Error);
10543        let error = parse_error(&response[0]);
10544        assert_eq!(error["code"], "invalid_manifest");
10545        assert!(error["message"]
10546            .as_str()
10547            .unwrap()
10548            .contains("rich_ui.schema"));
10549    }
10550
10551    #[test]
10552    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10553        let registry = Arc::new(Registry::default());
10554        let supervisor = SupervisorHandle::new();
10555        supervisor.open_swap("provider", "candidate-nonce".to_string());
10556        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10557        let connection = ConnectionId::new(218);
10558        let hello = handler
10559            .handle_control(
10560                connection,
10561                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10562            )
10563            .unwrap();
10564        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10565        assert!(registry.list_modules().unwrap().1.is_empty());
10566
10567        registry.promote_candidate("provider").unwrap().unwrap();
10568        let request = Frame::build(
10569            FrameType::Request,
10570            control_flags(),
10571            0,
10572            0,
10573            219,
10574            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10575        )
10576        .unwrap();
10577        let response = handler.handle_catalog_list(request, None).unwrap();
10578        let ClientControlResponse::CatalogList { modules, .. } =
10579            serde_json::from_slice(&response[0].body).unwrap()
10580        else {
10581            panic!("expected catalog.list");
10582        };
10583        assert_eq!(modules.len(), 1);
10584        assert!(modules[0].nonce_verified);
10585    }
10586
10587    #[test]
10588    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10589        for provenance_json in [
10590            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10591            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10592        ] {
10593            let registry = Arc::new(Registry::default());
10594            let handler = ControlHandler::new(registry);
10595            let hello = Frame::build(
10596                FrameType::Hello,
10597                control_flags(),
10598                0,
10599                0,
10600                220,
10601                serde_json::to_vec(&ModuleHelloBody {
10602                    manifest: ModuleManifest::builder("built", "0.1.0")
10603                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10604                        .build(),
10605                    protocol_ver: PROTOCOL_VERSION,
10606                    control_ops: None,
10607                    launch_nonce: None,
10608                })
10609                .unwrap(),
10610            )
10611            .unwrap();
10612            assert_eq!(
10613                handler
10614                    .handle_control(ConnectionId::new(220), hello)
10615                    .unwrap()[0]
10616                    .header
10617                    .ty,
10618                FrameType::HelloAck
10619            );
10620            let request = Frame::build(
10621                FrameType::Request,
10622                control_flags(),
10623                0,
10624                0,
10625                221,
10626                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10627            )
10628            .unwrap();
10629            let response = handler.handle_catalog_list(request, None).unwrap();
10630            let ClientControlResponse::CatalogList { modules, .. } =
10631                serde_json::from_slice(&response[0].body).unwrap()
10632            else {
10633                panic!("expected catalog.list");
10634            };
10635            assert_eq!(modules.len(), 1);
10636            assert_eq!(
10637                serde_json::to_vec(&modules[0]).unwrap(),
10638                format!(
10639                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10640                )
10641                .as_bytes()
10642            );
10643        }
10644    }
10645
10646    #[test]
10647    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10648        let registry = Arc::new(Registry::default());
10649        let handler = ControlHandler::new(Arc::clone(&registry));
10650        let hello = Frame::build(
10651            FrameType::Hello,
10652            control_flags(),
10653            0,
10654            0,
10655            220,
10656            serde_json::to_vec(&ModuleHelloBody {
10657                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10658                protocol_ver: PROTOCOL_VERSION,
10659                control_ops: None,
10660                launch_nonce: Some("unverified".to_string()),
10661            })
10662            .unwrap(),
10663        )
10664        .unwrap();
10665        assert_eq!(
10666            handler
10667                .handle_control(ConnectionId::new(220), hello)
10668                .unwrap()[0]
10669                .header
10670                .ty,
10671            FrameType::HelloAck
10672        );
10673        let request = Frame::build(
10674            FrameType::Request,
10675            control_flags(),
10676            0,
10677            0,
10678            221,
10679            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10680        )
10681        .unwrap();
10682        let response = handler.handle_catalog_list(request, None).unwrap();
10683        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10684        assert!(wire["modules"][0].get("provenance").is_none());
10685        let ClientControlResponse::CatalogList { modules, .. } =
10686            serde_json::from_slice(&response[0].body).unwrap()
10687        else {
10688            panic!("expected catalog.list");
10689        };
10690        assert_eq!(
10691            serde_json::to_vec(&modules[0]).unwrap(),
10692            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10693        );
10694        assert!(!modules[0].nonce_verified);
10695        assert!(modules[0].provenance.is_none());
10696    }
10697
10698    #[test]
10699    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10700        let registry = Arc::new(Registry::default());
10701        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10702            [("vault".to_string(), true), ("squatter".to_string(), true)],
10703            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10704        );
10705        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10706        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10707            provides: vec!["credentials-provider/v1".to_string()],
10708            requires: Vec::new(),
10709            must_never_reach: Vec::new(),
10710        });
10711        let frame = Frame::build(
10712            FrameType::Hello,
10713            control_flags(),
10714            0,
10715            0,
10716            77,
10717            serde_json::to_vec(&ModuleHelloBody {
10718                manifest: squatter,
10719                protocol_ver: PROTOCOL_VERSION,
10720                control_ops: None,
10721                launch_nonce: None,
10722            })
10723            .expect("HELLO serializes"),
10724        )
10725        .expect("HELLO frame builds");
10726        let response = handler
10727            .handle_control(ConnectionId::new(77), frame)
10728            .expect("reserved claim receives a typed refusal");
10729        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10730        assert_eq!(
10731            registry
10732                .active_registration_count()
10733                .expect("registry reads"),
10734            0,
10735            "a reserved capability refusal must not leave a catalog entry"
10736        );
10737    }
10738
10739    #[test]
10740    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10741        for settlement in ["timeout", "inconclusive", "drop"] {
10742            let breakers = RouteBindBreakers::default();
10743            let RouteBindAdmission::Admitted {
10744                guard: mut old,
10745                probe: false,
10746            } = breakers.admit("prov")
10747            else {
10748                panic!("ordinary relay admitted")
10749            };
10750            let RouteBindAdmission::Admitted {
10751                guard: mut opener, ..
10752            } = breakers.admit("prov")
10753            else {
10754                panic!("second relay admitted")
10755            };
10756            assert!(
10757                !opener
10758                    .record_timeout(1, Duration::ZERO)
10759                    .unwrap()
10760                    .reopened_after_probe
10761            );
10762            let RouteBindAdmission::Admitted {
10763                guard: mut probe,
10764                probe: true,
10765            } = breakers.admit("prov")
10766            else {
10767                panic!("one cooldown probe admitted")
10768            };
10769            match settlement {
10770                "timeout" => assert!(
10771                    !old.record_timeout(1, Duration::ZERO)
10772                        .unwrap()
10773                        .reopened_after_probe
10774                ),
10775                "inconclusive" => old.record_inconclusive(),
10776                "drop" => drop(old),
10777                _ => unreachable!(),
10778            }
10779            assert!(
10780                matches!(
10781                    breakers.admit("prov"),
10782                    RouteBindAdmission::Refused {
10783                        probe_in_flight: true,
10784                        ..
10785                    }
10786                ),
10787                "{settlement} of a pre-open relay cannot release the real probe"
10788            );
10789            assert!(
10790                probe
10791                    .record_timeout(1, Duration::ZERO)
10792                    .unwrap()
10793                    .reopened_after_probe
10794            );
10795            assert!(matches!(
10796                breakers.admit("prov"),
10797                RouteBindAdmission::Admitted { probe: true, .. }
10798            ));
10799        }
10800        let breakers = RouteBindBreakers::default();
10801        let admit = || match breakers.admit("prov") {
10802            RouteBindAdmission::Admitted { guard, .. } => guard,
10803            _ => panic!("relay admitted"),
10804        };
10805        admit().record_timeout(1, Duration::ZERO);
10806        let mut old_probe = admit();
10807        breakers.reset_for_new_module_connection("prov");
10808        admit().record_timeout(1, Duration::ZERO);
10809        let _new_probe = admit();
10810        old_probe.record_inconclusive();
10811        assert!(matches!(
10812            breakers.admit("prov"),
10813            RouteBindAdmission::Refused {
10814                probe_in_flight: true,
10815                ..
10816            }
10817        ));
10818    }
10819
10820    #[tokio::test]
10821    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10822        for candidate in [false, true] {
10823            let registry = Arc::new(Registry::default());
10824            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10825                [("vault".to_string(), true), ("squatter".to_string(), true)],
10826                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10827            );
10828            let conn = ConnectionId::new(77);
10829            let (ctx, mut rx) = route_ctx(conn);
10830            let initial = capability_manifest("squatter", &[], &[]);
10831            if candidate {
10832                registry
10833                    .register_candidate_with_control_ops(
10834                        initial.clone(),
10835                        PROTOCOL_VERSION,
10836                        conn,
10837                        module_baseline_control_ops(),
10838                    )
10839                    .unwrap();
10840                handler
10841                    .forwarding
10842                    .register_candidate_module_connection(
10843                        conn,
10844                        "squatter".to_string(),
10845                        PROTOCOL_VERSION,
10846                        manifest_concurrency(&initial),
10847                        ctx.egress.clone(),
10848                    )
10849                    .unwrap();
10850            } else {
10851                hello_via_sink(
10852                    &handler,
10853                    &ctx,
10854                    &mut rx,
10855                    hello_frame_with_manifest(initial.clone(), 1),
10856                )
10857                .await;
10858            }
10859            let response = handler
10860                .handle_control_frame(
10861                    &ctx,
10862                    catalog_update_with_capabilities_frame(
10863                        2,
10864                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
10865                            .capabilities
10866                            .unwrap(),
10867                    ),
10868                )
10869                .await
10870                .unwrap();
10871            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10872            assert_eq!(
10873                registry
10874                    .get_module_by_connection(conn)
10875                    .unwrap()
10876                    .unwrap()
10877                    .manifest,
10878                initial
10879            );
10880        }
10881    }
10882
10883    #[test]
10884    fn server_describe_surfaces_required_capability_verdict_fields() {
10885        let registry = Arc::new(Registry::default());
10886        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10887            [
10888                ("consumer".to_string(), true),
10889                ("provider".to_string(), false),
10890            ],
10891            BTreeMap::new(),
10892        );
10893        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10894        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10895            provides: Vec::new(),
10896            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10897                capability: "credentials-provider/v1".to_string(),
10898                need: subc_protocol::manifest::CapabilityNeed::Required,
10899            }],
10900            must_never_reach: Vec::new(),
10901        });
10902        let hello = Frame::build(
10903            FrameType::Hello,
10904            control_flags(),
10905            0,
10906            0,
10907            78,
10908            serde_json::to_vec(&ModuleHelloBody {
10909                manifest: consumer,
10910                protocol_ver: PROTOCOL_VERSION,
10911                control_ops: None,
10912                launch_nonce: None,
10913            })
10914            .expect("HELLO serializes"),
10915        )
10916        .expect("HELLO frame builds");
10917        handler
10918            .handle_control(ConnectionId::new(78), hello)
10919            .expect("consumer registers");
10920        let describe = Frame::build(
10921            FrameType::Request,
10922            control_flags(),
10923            0,
10924            0,
10925            79,
10926            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10927                .expect("request serializes"),
10928        )
10929        .expect("describe frame builds");
10930        let response = handler
10931            .handle_server_describe(describe)
10932            .expect("server.describe succeeds");
10933        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10934        let requirement = &rendered["capability_requirements"][0];
10935        assert_eq!(requirement["consumer"], "consumer");
10936        assert_eq!(requirement["verdict"], "never_provided");
10937        assert_eq!(requirement["episode_seq"], 1);
10938        assert_eq!(requirement["config_satisfiable"], false);
10939        assert_eq!(requirement["runtime_available"], false);
10940        assert!(requirement["detail"]
10941            .as_str()
10942            .expect("detail string")
10943            .contains("credentials-provider/v1"));
10944    }
10945
10946    /// Register a consumer that requires `project-identity/v1` and the provider
10947    /// of it, optionally mark daemon shutdown as begun, then deregister the
10948    /// provider as its connection closes. Returns the `capability.requirement`
10949    /// events that deregistration logged and the verdict the evaluator then
10950    /// holds for the requirement (the one `route.open` refusals read).
10951    fn deregister_required_provider(
10952        daemon_shutdown_begun: bool,
10953    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10954        const CAPABILITY: &str = "project-identity/v1";
10955        let registry = Arc::new(Registry::default());
10956        let daemon_shutdown = DaemonShutdownFlag::default();
10957        let handler = ControlHandler::new(Arc::clone(&registry))
10958            .with_capability_config(
10959                [
10960                    ("consumer".to_string(), true),
10961                    ("provider".to_string(), true),
10962                ],
10963                BTreeMap::new(),
10964            )
10965            .with_daemon_shutdown_flag(daemon_shutdown.clone());
10966        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10967        consumer.capabilities = Some(CapabilityDeclarations {
10968            provides: Vec::new(),
10969            requires: vec![subc_protocol::manifest::CapabilityRequirement {
10970                capability: CAPABILITY.to_string(),
10971                need: subc_protocol::manifest::CapabilityNeed::Required,
10972            }],
10973            must_never_reach: Vec::new(),
10974        });
10975        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10976        let provider_conn = ConnectionId::new(182);
10977        for (conn, manifest, corr) in [
10978            (ConnectionId::new(181), consumer, 181),
10979            (provider_conn, provider, 182),
10980        ] {
10981            let hello = handler
10982                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10983                .expect("module registers");
10984            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10985        }
10986        assert_eq!(
10987            handler.capability_evaluator.verdict("consumer", CAPABILITY),
10988            Some(CapabilityVerdict::Provided),
10989            "the registered provider must satisfy the requirement first"
10990        );
10991        if daemon_shutdown_begun {
10992            daemon_shutdown.set_for_test();
10993        }
10994
10995        let capture = EventCapture::default();
10996        let _subscriber =
10997            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10998        handler
10999            .cleanup_connection(provider_conn)
11000            .expect("provider deregisters");
11001        let events = capture
11002            .events()
11003            .into_iter()
11004            .filter(|event| {
11005                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11006            })
11007            .collect();
11008        (
11009            events,
11010            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11011        )
11012    }
11013
11014    #[test]
11015    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11016        let (events, verdict) = deregister_required_provider(true);
11017        assert_eq!(
11018            verdict,
11019            Some(CapabilityVerdict::NeverProvided),
11020            "shutdown must not change the verdict route.open refusals read"
11021        );
11022        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11023        let event = &events[0];
11024        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11025        assert_eq!(
11026            event.fields.get("cause").map(String::as_str),
11027            Some("\"daemon_shutdown\""),
11028            "{event:?}"
11029        );
11030        assert_eq!(
11031            event.fields.get("verdict").map(String::as_str),
11032            Some("\"never_provided\""),
11033            "{event:?}"
11034        );
11035    }
11036
11037    #[test]
11038    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11039        let (events, verdict) = deregister_required_provider(false);
11040        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11041        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11042        let event = &events[0];
11043        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11044        assert_eq!(event.fields.get("cause"), None, "{event:?}");
11045        assert_eq!(
11046            event.fields.get("verdict").map(String::as_str),
11047            Some("\"never_provided\""),
11048            "{event:?}"
11049        );
11050    }
11051
11052    #[test]
11053    fn catalog_list_omits_capabilities_for_legacy_manifest() {
11054        let registry = Arc::new(Registry::default());
11055        let handler = ControlHandler::new(Arc::clone(&registry));
11056        let hello = handler
11057            .handle_control(
11058                ConnectionId::new(101),
11059                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11060            )
11061            .expect("legacy HELLO registers");
11062        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11063
11064        let request = Frame::build(
11065            FrameType::Request,
11066            control_flags(),
11067            0,
11068            0,
11069            102,
11070            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11071                .expect("catalog request serializes"),
11072        )
11073        .expect("catalog request frame builds");
11074        let response = handler
11075            .handle_catalog_list(request, None)
11076            .expect("catalog list succeeds");
11077        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11078        assert!(
11079            body["modules"][0].get("capabilities").is_none(),
11080            "legacy manifest must retain an absent capabilities field on catalog.list"
11081        );
11082    }
11083
11084    #[test]
11085    fn hello_ack_omits_storage_when_no_storage_config() {
11086        let registry = Arc::new(Registry::default());
11087        let handler = ControlHandler::new(Arc::clone(&registry));
11088        let responses = handler
11089            .handle_control(
11090                ConnectionId::new(1),
11091                hello_frame("aft", PROTOCOL_VERSION, 7),
11092            )
11093            .unwrap();
11094        let ack = parse_ack(&responses[0]);
11095        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11096        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11097    }
11098
11099    #[tokio::test]
11100    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11101        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11102        let registry = Arc::new(Registry::default());
11103        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
11104        let responses = handler
11105            .handle_control(
11106                ConnectionId::new(1),
11107                hello_frame("aft", PROTOCOL_VERSION, 7),
11108            )
11109            .unwrap();
11110        let ack = parse_ack(&responses[0]);
11111        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11112
11113        let described = handler
11114            .handle_control_frame(
11115                &route_ctx(ConnectionId::new(2)).0,
11116                Frame::build(
11117                    FrameType::Request,
11118                    control_flags(),
11119                    0,
11120                    0,
11121                    9,
11122                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11123                )
11124                .unwrap(),
11125            )
11126            .await
11127            .unwrap();
11128        let ClientControlResponse::ServerDescribe { machine_id, .. } =
11129            serde_json::from_slice(&described[0].body).unwrap()
11130        else {
11131            panic!("server.describe answered with another shape");
11132        };
11133        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11134    }
11135
11136    #[test]
11137    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11138        // With a central sqlite storage policy, each registering module gets its
11139        // own resolved descriptor in HELLO_ACK, keyed by its module id.
11140        let registry = Arc::new(Registry::default());
11141        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
11142            crate::daemon_config::StorageConfig::Sqlite {
11143                data_home: std::path::PathBuf::from("/data"),
11144            },
11145        ));
11146
11147        let responses = handler
11148            .handle_control(
11149                ConnectionId::new(1),
11150                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11151            )
11152            .unwrap();
11153        let ack = parse_ack(&responses[0]);
11154        assert_eq!(
11155            ack.storage,
11156            Some(serde_json::json!({
11157                "module_id": "alfonso-routing",
11158                "storage_namespace": "default",
11159                "isolation": { "kind": "module" },
11160                "backend": {
11161                    "backend": "sqlite",
11162                    "path": "/data/cortexkit/alfonso-routing/store.db"
11163                }
11164            })),
11165            "the delivered descriptor is the module's own sqlite store path"
11166        );
11167    }
11168
11169    #[test]
11170    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11171        let registry = Arc::new(Registry::default());
11172        let handler = ControlHandler::new(Arc::clone(&registry));
11173        let conn = ConnectionId::new(1);
11174        let responses = handler
11175            .handle_control(
11176                conn,
11177                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11178            )
11179            .unwrap();
11180        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11181        let registration = registry.get_module("aft").unwrap().unwrap();
11182        assert_eq!(registration.control_ops, module_baseline_control_ops());
11183
11184        let frame =
11185            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11186        assert!(handler
11187            .guard_module_control_op(&frame, "aft", "route.bind")
11188            .unwrap()
11189            .is_none());
11190        let error = handler
11191            .guard_module_control_op(&frame, "aft", "test.synthetic")
11192            .unwrap()
11193            .expect("synthetic ungranted op should be rejected");
11194        assert_eq!(error.header.ty, FrameType::Error);
11195        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11196    }
11197
11198    #[test]
11199    fn hello_control_ops_some_adds_optional_grants() {
11200        let registry = Arc::new(Registry::default());
11201        let handler = ControlHandler::new(Arc::clone(&registry));
11202        handler
11203            .handle_control(
11204                ConnectionId::new(1),
11205                hello_frame_with_control_ops(
11206                    "aft",
11207                    PROTOCOL_VERSION,
11208                    7,
11209                    Some(vec![
11210                        "future.synthetic".to_string(),
11211                        "route.bind".to_string(),
11212                    ]),
11213                ),
11214            )
11215            .unwrap();
11216        let registration = registry.get_module("aft").unwrap().unwrap();
11217        assert_eq!(
11218            registration.control_ops,
11219            vec![
11220                "route.bind".to_string(),
11221                "route.status".to_string(),
11222                "future.synthetic".to_string(),
11223            ]
11224        );
11225        let frame =
11226            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11227        assert!(handler
11228            .guard_module_control_op(&frame, "aft", "future.synthetic")
11229            .unwrap()
11230            .is_none());
11231    }
11232
11233    #[tokio::test]
11234    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11235        let registry = Arc::new(Registry::default());
11236        let forwarding = Arc::new(ForwardingTable::default());
11237        let handler =
11238            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11239        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11240        hello_via_sink(
11241            &handler,
11242            &module_ctx,
11243            &mut module_rx,
11244            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11245        )
11246        .await;
11247
11248        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11249        let responses = handler
11250            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11251            .await
11252            .unwrap();
11253        assert_eq!(responses.len(), 1);
11254        assert_eq!(responses[0].header.ty, FrameType::Error);
11255        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11256        assert!(module_rx.try_recv().is_err());
11257    }
11258
11259    #[tokio::test]
11260    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11261        let registry = Arc::new(Registry::default());
11262        let forwarding = Arc::new(ForwardingTable::default());
11263        let handler =
11264            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11265        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11266        hello_via_sink(
11267            &handler,
11268            &module_ctx,
11269            &mut module_rx,
11270            hello_frame_with_control_ops(
11271                "aft",
11272                PROTOCOL_VERSION,
11273                7,
11274                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11275            ),
11276        )
11277        .await;
11278
11279        let project_root = unique_project_root("demux");
11280        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11281        let route_handler = handler.clone();
11282        let route_task = tokio::spawn(async move {
11283            route_handler
11284                .handle_control_frame(
11285                    &route_client_ctx,
11286                    route_open_frame(100, "aft", project_root),
11287                )
11288                .await
11289                .unwrap()
11290        });
11291        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11292            .await
11293            .unwrap()
11294            .unwrap();
11295        assert!(matches!(
11296            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11297            ModuleControlRequest::RouteBind { .. }
11298        ));
11299
11300        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11301        let health_handler = handler.clone();
11302        let health_task = tokio::spawn(async move {
11303            health_handler
11304                .handle_control_frame(
11305                    &health_client_ctx,
11306                    supervisor_health_probe_frame(101, "aft"),
11307                )
11308                .await
11309                .unwrap()
11310        });
11311        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11312            .await
11313            .unwrap()
11314            .unwrap();
11315        assert_eq!(
11316            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11317            ModuleControlRequest::HealthCheck {}
11318        );
11319
11320        handler
11321            .handle_control_frame(
11322                &module_ctx,
11323                health_response(health_frame.header.corr, HealthStatus::Degraded),
11324            )
11325            .await
11326            .unwrap();
11327        let health_response = health_task.await.unwrap();
11328        assert_eq!(health_response.len(), 1);
11329        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11330            ClientControlResponse::SupervisorHealthProbe {
11331                module_id,
11332                status,
11333                detail,
11334                metrics,
11335            } => {
11336                assert_eq!(module_id, "aft");
11337                assert_eq!(status, HealthStatus::Degraded);
11338                assert_eq!(detail.as_deref(), Some("warming"));
11339                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11340            }
11341            other => panic!("unexpected health response: {other:?}"),
11342        }
11343
11344        handler
11345            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11346            .await
11347            .unwrap();
11348        let route_response = route_task.await.unwrap();
11349        assert!(route_response.is_empty());
11350        let published = route_client_rx.recv().await.unwrap();
11351        assert!(matches!(
11352            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11353            ClientControlResponse::RouteOpen { .. }
11354        ));
11355    }
11356
11357    /// Start one `route.open` on `client_connection` and return its still-running
11358    /// handler task together with the `route.bind` the module received for it.
11359    /// The handler blocks until the module answers, so it has to run as a task
11360    /// while the test drives the module side.
11361    async fn relay_route_open(
11362        handler: &ControlHandler,
11363        client_connection: ConnectionId,
11364        client_egress: &FrameSink,
11365        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11366        corr: u64,
11367        module_id: &str,
11368        project_root_label: &str,
11369    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11370        let ctx = RouteCtx {
11371            connection_id: client_connection,
11372            egress: client_egress.clone(),
11373        };
11374        let handler = handler.clone();
11375        let project_root = unique_project_root(project_root_label);
11376        let module_id = module_id.to_string();
11377        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11378        let task = tokio::spawn(async move {
11379            let _guard = tracing::dispatcher::set_default(&dispatch);
11380            handler
11381                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11382                .await
11383                .unwrap()
11384        });
11385        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11386            .await
11387            .expect("module receives the relayed route.bind")
11388            .expect("module egress is open");
11389        (task, bind.frame)
11390    }
11391
11392    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11393        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11394            ModuleControlRequest::RouteBind {
11395                route_channel,
11396                epoch,
11397                ..
11398            } => (route_channel, epoch),
11399            other => panic!("expected a route.bind request, got {other:?}"),
11400        }
11401    }
11402
11403    fn published_route(frame: &Frame) -> (u16, u32) {
11404        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11405            ClientControlResponse::RouteOpen {
11406                route_channel,
11407                route_epoch,
11408                ..
11409            } => (route_channel, route_epoch),
11410            other => panic!("expected a route.open response, got {other:?}"),
11411        }
11412    }
11413
11414    /// Reproduction of a production outage. A client had `route.open`s in
11415    /// flight to a module and was already marked closing -- its egress had refused a
11416    /// module frame, so the daemon asked its connection to end -- while its sink
11417    /// was still open. When the module acked those binds, the daemon refused to
11418    /// commit a route for a closing client, and that refusal was returned from
11419    /// the MODULE connection's frame handler, where a router error that has no
11420    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
11421    /// the supervisor correctly did not respawn a clean exit, and every seat lost
11422    /// its tools for hours -- one client's teardown took down a connection
11423    /// carrying ~170 other routes.
11424    ///
11425    /// The window is opened here by calling the production path that opens it
11426    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
11427    /// state that matters is "in `closing_connections`, sink still open, relay
11428    /// still pending", and it lasts only from the close request until the
11429    /// connection loop reacts to it; a socket-level test can flood a client into
11430    /// that escalation but cannot pin the module's ack inside the window. Closing
11431    /// the socket instead takes the other path entirely -- connection teardown
11432    /// removes the pending relay under the same lock, so the ack finds nothing.
11433    #[tokio::test]
11434    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11435        let registry = Arc::new(Registry::default());
11436        let forwarding = Arc::new(ForwardingTable::default());
11437        let handler =
11438            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11439
11440        let module_connection = ConnectionId::new(30);
11441        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11442        hello_via_sink(
11443            &handler,
11444            &module_ctx,
11445            &mut module_rx,
11446            hello_frame("aft", PROTOCOL_VERSION, 7),
11447        )
11448        .await;
11449
11450        let dying_client = ConnectionId::new(31);
11451        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11452
11453        // A published route on the dying client. The escalation below only marks
11454        // a connection closing for a route it has already published.
11455        let (first_task, first_bind) = relay_route_open(
11456            &handler,
11457            dying_client,
11458            &dying_ctx.egress,
11459            &mut module_rx,
11460            100,
11461            "aft",
11462            "closing-first",
11463        )
11464        .await;
11465        handler
11466            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11467            .await
11468            .unwrap();
11469        assert!(first_task.await.unwrap().is_empty());
11470        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11471
11472        // A second route.open from the same client, relayed and awaiting its ack.
11473        let (second_task, second_bind) = relay_route_open(
11474            &handler,
11475            dying_client,
11476            &dying_ctx.egress,
11477            &mut module_rx,
11478            101,
11479            "aft",
11480            "closing-second",
11481        )
11482        .await;
11483        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11484
11485        // The window: the client is closing, its sink is still open, and its
11486        // second bind is still pending.
11487        assert!(forwarding
11488            .escalate_client_delivery_failure(
11489                dying_client,
11490                first_channel,
11491                first_epoch,
11492                CloseReason::new(
11493                    "module_to_client_delivery_failed",
11494                    "client egress refused a module frame",
11495                ),
11496                crate::forwarding::UndeliveredFrame {
11497                    module_id: None,
11498                    sink: &dying_ctx.egress,
11499                },
11500            )
11501            .unwrap());
11502        assert!(!dying_ctx.egress.is_closed());
11503
11504        // The frame that used to end the module connection.
11505        let ack = handler
11506            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11507            .await;
11508        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11509        if module_loop_error.is_some() {
11510            // What the server's connection loop does with a router error that has
11511            // no ERROR-frame translation: end the connection, which releases the
11512            // module's registration and every route on it.
11513            handler.cleanup_connection(module_connection).unwrap();
11514        }
11515        // Read the module's next frame before opening the co-tenant's route, so
11516        // the GOODBYE assertion below is about THIS ack and not about later
11517        // traffic. `None` means the module was told nothing.
11518        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11519            .await
11520            .ok()
11521            .flatten();
11522
11523        // 1. The module connection is still registered.
11524        assert!(
11525            registry
11526                .get_module_by_connection(module_connection)
11527                .unwrap()
11528                .is_some(),
11529            "one client's closing connection ended the shared module connection: \
11530             {module_loop_error:?}"
11531        );
11532        // ...and still serving: another client can open and use a route on it.
11533        let cotenant = ConnectionId::new(32);
11534        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11535        let (cotenant_task, cotenant_bind) = relay_route_open(
11536            &handler,
11537            cotenant,
11538            &cotenant_ctx.egress,
11539            &mut module_rx,
11540            102,
11541            "aft",
11542            "closing-cotenant",
11543        )
11544        .await;
11545        handler
11546            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11547            .await
11548            .unwrap();
11549        assert!(cotenant_task.await.unwrap().is_empty());
11550        let (cotenant_channel, cotenant_epoch) =
11551            published_route(&cotenant_rx.recv().await.unwrap());
11552        assert!(matches!(
11553            forwarding
11554                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11555                .unwrap(),
11556            DataRoute::Client(DataRouteState::Bound(_))
11557        ));
11558
11559        // 2. The module was told to drop the binding it created for the route
11560        //    that will never be published.
11561        let goodbye = post_ack_module_frame
11562            .expect("module receives a GOODBYE for the abandoned route channel");
11563        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11564        assert_eq!(goodbye.header.channel, abandoned_channel);
11565        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11566
11567        // 3. The dying client received nothing: no route was ever published to
11568        //    it. Its route.open is answered as unavailable, which the connection
11569        //    loop would write to a socket that is already going away.
11570        assert!(dying_rx.try_recv().is_err());
11571        let second_response = second_task.await.unwrap();
11572        assert_eq!(second_response.len(), 1);
11573        assert_eq!(
11574            parse_error(&second_response[0])["code"],
11575            "target_unavailable"
11576        );
11577    }
11578
11579    /// One `catalog.list` as a client sees it: the generation and the listed
11580    /// module ids with their readiness.
11581    async fn catalog_snapshot(
11582        handler: &ControlHandler,
11583        client_ctx: &RouteCtx,
11584        corr: u64,
11585    ) -> (u64, Vec<(String, bool)>) {
11586        let body =
11587            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11588        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11589        let reply = handler
11590            .handle_control_frame(client_ctx, request)
11591            .await
11592            .unwrap()
11593            .pop()
11594            .unwrap();
11595        let ClientControlResponse::CatalogList {
11596            generation,
11597            modules,
11598            ..
11599        } = serde_json::from_slice(&reply.body).unwrap()
11600        else {
11601            panic!("not a catalog.list reply");
11602        };
11603        (
11604            generation,
11605            modules
11606                .into_iter()
11607                .map(|entry| (entry.module_id, entry.ready))
11608                .collect(),
11609        )
11610    }
11611
11612    /// A handler whose HELLO for module "aft" on connection 40 is parked
11613    /// between reserving its registry entry and installing its forwarding
11614    /// endpoint. Returns the handler, the two-party barrier (wait once to know
11615    /// the HELLO is parked, once more to release it), the HELLO's thread, and
11616    /// the module's context and receiver.
11617    fn hello_held_before_forwarding() -> (
11618        ControlHandler,
11619        Arc<std::sync::Barrier>,
11620        std::thread::JoinHandle<Vec<Frame>>,
11621        RouteCtx,
11622        mpsc::Receiver<crate::router::OutboundFrame>,
11623    ) {
11624        let hold = Arc::new(std::sync::Barrier::new(2));
11625        let handler = ControlHandler::with_forwarding(
11626            Arc::new(Registry::default()),
11627            Arc::new(ForwardingTable::default()),
11628        )
11629        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11630        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11631        let hello = {
11632            let handler = handler.clone();
11633            let module_ctx = module_ctx.clone();
11634            std::thread::spawn(move || {
11635                handler
11636                    .handle_hello(
11637                        module_ctx.connection_id,
11638                        Some(module_ctx.egress.clone()),
11639                        hello_frame("aft", PROTOCOL_VERSION, 7),
11640                    )
11641                    .unwrap()
11642            })
11643        };
11644        (handler, hold, hello, module_ctx, module_rx)
11645    }
11646
11647    /// While a HELLO has reserved its registry entry but its forwarding
11648    /// endpoint is not installed yet, `route.open` to the module can only be
11649    /// refused. `catalog.list` must not list the module in that window: a
11650    /// client that opens a route as soon as it sees a module listed and ready
11651    /// would be refused for a module the daemon had just advertised. The
11652    /// refusal in the window stays the retryable `target_unavailable`, never
11653    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
11654    /// Once the HELLO completes, the module is listed and a route to it opens.
11655    #[tokio::test]
11656    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11657        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11658        let client = ConnectionId::new(41);
11659        let (client_ctx, mut client_rx) = route_ctx(client);
11660
11661        hold.wait();
11662        // Parked: the registry holds "aft", forwarding has no endpoint for it.
11663        assert!(handler.registry.get_module("aft").unwrap().is_some());
11664        assert!(!handler
11665            .forwarding
11666            .has_live_module_connection("aft")
11667            .unwrap());
11668        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11669        assert!(
11670            listed.is_empty(),
11671            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11672        );
11673        let refused = handler
11674            .handle_control_frame(
11675                &client_ctx,
11676                route_open_frame(301, "aft", unique_project_root("listed-window")),
11677            )
11678            .await
11679            .unwrap();
11680        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11681        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11682        let duplicate = handler
11683            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11684            .await
11685            .unwrap();
11686        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11687
11688        hold.wait();
11689        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11690        let ack = module_rx.try_recv().unwrap().frame;
11691        assert_eq!(ack.header.ty, FrameType::HelloAck);
11692        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11693        assert_eq!(listed, vec![("aft".to_string(), true)]);
11694
11695        let (open_task, bind) = relay_route_open(
11696            &handler,
11697            client,
11698            &client_ctx.egress,
11699            &mut module_rx,
11700            303,
11701            "aft",
11702            "listed-after",
11703        )
11704        .await;
11705        assert!(handler
11706            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11707            .await
11708            .unwrap()
11709            .is_empty());
11710        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11711        published_route(&client_rx.try_recv().unwrap().frame);
11712    }
11713
11714    /// A catalog watcher keys on the generation: when it changes, it re-reads
11715    /// the listing. So the generation must advance exactly when the module
11716    /// appears. If it advanced while the module was still hidden, a watcher
11717    /// would read the new generation without the module, and the module would
11718    /// later appear under that same generation with nothing telling the
11719    /// watcher to look again.
11720    #[tokio::test]
11721    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11722        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11723        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11724        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11725
11726        hold.wait();
11727        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11728        assert!(listed.is_empty());
11729        assert_eq!(
11730            during, before,
11731            "the generation moved while the module was not listed"
11732        );
11733
11734        hold.wait();
11735        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11736        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11737        assert_eq!(listed, vec![("aft".to_string(), true)]);
11738        assert_eq!(
11739            after,
11740            before + 1,
11741            "the module appeared without exactly one generation bump"
11742        );
11743    }
11744
11745    /// A HELLO whose forwarding endpoint cannot be installed (here the
11746    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11747    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11748    /// and the catalog generation where it was.
11749    #[tokio::test]
11750    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11751        let registry = Arc::new(Registry::default());
11752        let handler = ControlHandler::with_forwarding(
11753            Arc::clone(&registry),
11754            Arc::new(ForwardingTable::default()),
11755        );
11756        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11757        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11758        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11759        drop(module_rx);
11760
11761        let replies = handler
11762            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11763            .await
11764            .unwrap();
11765        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11766        assert!(registry.get_module("aft").unwrap().is_none());
11767        assert!(registry
11768            .get_module_by_connection(module_ctx.connection_id)
11769            .unwrap()
11770            .is_none());
11771        assert_eq!(
11772            catalog_snapshot(&handler, &client_ctx, 301).await,
11773            (before, Vec::new())
11774        );
11775    }
11776
11777    /// A handler whose `route.open` parks after queueing the route.bind and
11778    /// before marking it enqueued, with module "aft" registered on connection
11779    /// 40. Returns the two-party barrier that parks and releases it.
11780    async fn route_open_held_after_bind_send() -> (
11781        ControlHandler,
11782        Arc<tokio::sync::Barrier>,
11783        RouteCtx,
11784        mpsc::Receiver<crate::router::OutboundFrame>,
11785    ) {
11786        let hold = Arc::new(tokio::sync::Barrier::new(2));
11787        let handler = ControlHandler::with_forwarding(
11788            Arc::new(Registry::default()),
11789            Arc::new(ForwardingTable::default()),
11790        )
11791        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11792        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11793        hello_via_sink(
11794            &handler,
11795            &module_ctx,
11796            &mut module_rx,
11797            hello_frame("aft", PROTOCOL_VERSION, 7),
11798        )
11799        .await;
11800        (handler, hold, module_ctx, module_rx)
11801    }
11802
11803    /// A fast module acks the route.bind, and the reader commits the route,
11804    /// before `route.open` has marked the relay enqueued. The daemon used to
11805    /// read the missing relay as an abandoned bind and send the module a
11806    /// GOODBYE for the route it had just committed: the client was told its
11807    /// route was open, the module unbound the channel, and the client's first
11808    /// request was dropped on the module side as unbound.
11809    #[tokio::test]
11810    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11811        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11812        let client = ConnectionId::new(41);
11813        let (client_ctx, mut client_rx) = route_ctx(client);
11814
11815        let (open_task, bind) = relay_route_open(
11816            &handler,
11817            client,
11818            &client_ctx.egress,
11819            &mut module_rx,
11820            200,
11821            "aft",
11822            "acked-before-mark",
11823        )
11824        .await;
11825        let (module_channel, module_epoch) = route_bind_channel(&bind);
11826        // route.open is now parked between queueing the bind and marking it.
11827        hold.wait().await;
11828
11829        // The module answers inside that window, and the route commits.
11830        assert!(handler
11831            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11832            .await
11833            .unwrap()
11834            .is_empty());
11835        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11836
11837        hold.wait().await;
11838        assert!(
11839            open_task.await.unwrap().is_empty(),
11840            "route.open answers accepted"
11841        );
11842
11843        // Every frame route.open could have sent is already queued, because it
11844        // has returned; the module must have been sent nothing.
11845        match module_rx.try_recv() {
11846            Err(mpsc::error::TryRecvError::Empty) => {}
11847            Ok(outbound) => panic!(
11848                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11849                outbound.frame.header.ty,
11850                outbound.frame.header.channel,
11851                outbound.frame.header.epoch
11852            ),
11853            Err(err) => panic!("module egress closed: {err}"),
11854        }
11855
11856        // The client's first request on the route is the next thing the module
11857        // reads, on the binding it just created.
11858        let router = Router::with_control_handler(Arc::new(handler.clone()));
11859        let request = Frame::build(
11860            FrameType::Request,
11861            Flags::new(true, Priority::Interactive, false),
11862            client_channel,
11863            client_epoch,
11864            201,
11865            b"first request".to_vec(),
11866        )
11867        .unwrap();
11868        router
11869            .route_for_connection(&client_ctx, request)
11870            .await
11871            .unwrap();
11872        let delivered = module_rx.try_recv().unwrap().frame;
11873        assert_eq!(delivered.header.ty, FrameType::Request);
11874        assert_eq!(
11875            (delivered.header.channel, delivered.header.epoch),
11876            (module_channel, module_epoch)
11877        );
11878        assert_eq!(delivered.body, b"first request");
11879    }
11880
11881    /// The client goes away while `route.open` is still queueing the bind, so
11882    /// the relay is dropped with no module reply. The teardown cannot send the
11883    /// GOODBYE itself (it could reach the module ahead of the bind), so
11884    /// `route.open` must send it once the bind is queued; otherwise the module
11885    /// keeps a binding the daemon will never commit.
11886    #[tokio::test]
11887    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11888        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11889        let client = ConnectionId::new(42);
11890        let (client_ctx, _client_rx) = route_ctx(client);
11891
11892        let (open_task, bind) = relay_route_open(
11893            &handler,
11894            client,
11895            &client_ctx.egress,
11896            &mut module_rx,
11897            300,
11898            "aft",
11899            "released-before-mark",
11900        )
11901        .await;
11902        let (module_channel, module_epoch) = route_bind_channel(&bind);
11903        hold.wait().await;
11904
11905        handler.cleanup_connection(client).unwrap();
11906        assert!(
11907            module_rx.try_recv().is_err(),
11908            "client teardown must not GOODBYE a bind route.open is still queueing"
11909        );
11910
11911        hold.wait().await;
11912        let response = open_task.await.unwrap();
11913        assert_eq!(response.len(), 1);
11914        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11915
11916        let goodbye = module_rx
11917            .try_recv()
11918            .expect("module is told to drop the binding the daemon abandoned")
11919            .frame;
11920        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11921        assert_eq!(
11922            (goodbye.header.channel, goodbye.header.epoch),
11923            (module_channel, module_epoch)
11924        );
11925        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11926    }
11927
11928    /// The module acks inside the same window, but the client's egress closed
11929    /// first, so the reader refuses to commit. The ack proves the module holds
11930    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
11931    /// send a second one.
11932    #[tokio::test]
11933    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11934        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11935        let client = ConnectionId::new(43);
11936        let (client_ctx, client_rx) = route_ctx(client);
11937
11938        let (open_task, bind) = relay_route_open(
11939            &handler,
11940            client,
11941            &client_ctx.egress,
11942            &mut module_rx,
11943            400,
11944            "aft",
11945            "gone-client-before-mark",
11946        )
11947        .await;
11948        let (module_channel, module_epoch) = route_bind_channel(&bind);
11949        hold.wait().await;
11950
11951        drop(client_rx);
11952        assert!(client_ctx.egress.is_closed());
11953        handler
11954            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11955            .await
11956            .unwrap();
11957
11958        hold.wait().await;
11959        let response = open_task.await.unwrap();
11960        assert_eq!(response.len(), 1);
11961        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11962
11963        let goodbye = module_rx
11964            .try_recv()
11965            .expect("module is told to drop the binding it acked")
11966            .frame;
11967        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11968        assert_eq!(
11969            (goodbye.header.channel, goodbye.header.epoch),
11970            (module_channel, module_epoch)
11971        );
11972        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11973        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11974    }
11975
11976    /// The fence at the module-loop boundary, stated as its own contract: which
11977    /// forwarding failures are allowed to end the module connection that is being
11978    /// served. A `ConnectionClosing` naming some client is about that client, and
11979    /// a module connection is shared; the same error naming the module's own
11980    /// connection is about this connection and must stay fatal, as must failures
11981    /// that are about the forwarding table itself.
11982    #[test]
11983    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11984        let handler = ControlHandler::default();
11985        let module_connection = ConnectionId::new(30);
11986        let client_connection = ConnectionId::new(31);
11987
11988        handler
11989            .refuse_to_end_module_connection_for_a_client(
11990                module_connection,
11991                77,
11992                ForwardingError::ConnectionClosing {
11993                    connection_id: client_connection,
11994                },
11995            )
11996            .expect("a closing client must never end the module connection");
11997
11998        assert!(matches!(
11999            handler.refuse_to_end_module_connection_for_a_client(
12000                module_connection,
12001                78,
12002                ForwardingError::ConnectionClosing {
12003                    connection_id: module_connection,
12004                },
12005            ),
12006            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12007                connection_id
12008            })) if connection_id == module_connection
12009        ));
12010        assert!(matches!(
12011            handler.refuse_to_end_module_connection_for_a_client(
12012                module_connection,
12013                79,
12014                ForwardingError::Poisoned,
12015            ),
12016            Err(RouterError::Forwarding(ForwardingError::Poisoned))
12017        ));
12018        assert!(matches!(
12019            handler.refuse_to_end_module_connection_for_a_client(
12020                module_connection,
12021                80,
12022                ForwardingError::StaleModuleEndpoint,
12023            ),
12024            Err(RouterError::Forwarding(
12025                ForwardingError::StaleModuleEndpoint
12026            ))
12027        ));
12028    }
12029
12030    /// The spawn-attestation guard is what stops a connected module from claiming
12031    /// another module's identity and being stamped `Reserved` for it. Every other
12032    /// test that supplies a consumer_identity supplies a CORRECT one, because a
12033    /// correct one is what the rest of the flow needs -- so the guard's rejection
12034    /// branch was never the subject of an assertion, only its acceptance branch.
12035    ///
12036    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
12037    /// whole subc-core library suite green; only the forwarding integration tests
12038    /// notice, and they notice for unrelated reasons. This test exists so the
12039    /// refusal itself is asserted where the guard lives: it fails if the identity
12040    /// check stops refusing, which is the direction that matters, since a guard
12041    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
12042    #[tokio::test]
12043    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12044        let registry = Arc::new(Registry::default());
12045        let forwarding = Arc::new(ForwardingTable::default());
12046        let supervisor = SupervisorHandle::new();
12047        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12048        let handler =
12049            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12050                .with_supervisor(supervisor);
12051
12052        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12053        hello_via_sink(
12054            &handler,
12055            &target_ctx,
12056            &mut target_rx,
12057            hello_frame("target", PROTOCOL_VERSION, 1),
12058        )
12059        .await;
12060
12061        // A real supervised module id presenting the wrong nonce. This is the
12062        // impersonation case: the attacker knows a privileged module_id, which is
12063        // public, and guesses at the nonce, which is not.
12064        let wrong_nonce = handler
12065            .handle_control_frame(
12066                &route_ctx(ConnectionId::new(91)).0,
12067                route_open_frame_with_admission_facts(
12068                    20,
12069                    "target",
12070                    unique_project_root("admission-facts"),
12071                    Some(subc_control::ConsumerIdentity {
12072                        module_id: "fed".to_string(),
12073                        launch_nonce: "not-the-real-nonce".to_string(),
12074                    }),
12075                    None,
12076                ),
12077            )
12078            .await
12079            .unwrap();
12080        assert_eq!(
12081            parse_error(&wrong_nonce[0])["code"],
12082            "bad_consumer_identity",
12083            "a mismatched launch nonce must be refused, not stamped Reserved"
12084        );
12085
12086        // A module id the supervisor never spawned at all, so no nonce exists to
12087        // compare against. An implementation that treats "no record" as "nothing
12088        // to check" fails open here while passing the case above.
12089        let never_spawned = handler
12090            .handle_control_frame(
12091                &route_ctx(ConnectionId::new(92)).0,
12092                route_open_frame_with_admission_facts(
12093                    21,
12094                    "target",
12095                    unique_project_root("admission-facts"),
12096                    Some(subc_control::ConsumerIdentity {
12097                        module_id: "never-spawned".to_string(),
12098                        launch_nonce: "any-nonce".to_string(),
12099                    }),
12100                    None,
12101                ),
12102            )
12103            .await
12104            .unwrap();
12105        assert_eq!(
12106            parse_error(&never_spawned[0])["code"],
12107            "bad_consumer_identity",
12108            "an unspawned module_id must be refused rather than accepted for lack of a record"
12109        );
12110    }
12111
12112    /// The refusal test above proves the guard says NO. Nothing proved it can say
12113    /// YES, and the difference is not academic: replacing the whole authorization
12114    /// with `false` -- admitting no consumer identity at all, revoking Reserved
12115    /// standing for every supervised module in the fleet -- leaves 110 of the 111
12116    /// library tests GREEN. The one that notices does so by HANGING, because it
12117    /// waits for a bind that can no longer happen.
12118    ///
12119    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
12120    /// or flaky test, invites a RETRY rather than an investigation, and the retry
12121    /// hangs too and gets blamed on the runner. So a total revocation of the
12122    /// daemon's trust grant would have shipped behind a symptom nobody attributes
12123    /// to code.
12124    ///
12125    /// The bias is structural rather than accidental. A REFUSAL looks like a
12126    /// failure someone writes a test for; a GRANT looks like the happy path. Every
12127    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
12128    /// suite for that reason, and this one is the purest case in the daemon.
12129    ///
12130    /// This test asserts the EFFECT rather than the absence of an error: the module
12131    /// receives a RouteBind and it carries `Reserved` naming the attested module.
12132    /// A guard that admitted nobody would produce no bind at all; one that admitted
12133    /// everybody would stamp the wrong principal, which the refusal test catches.
12134    #[tokio::test]
12135    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12136        let registry = Arc::new(Registry::default());
12137        let forwarding = Arc::new(ForwardingTable::default());
12138        let supervisor = SupervisorHandle::new();
12139        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12140        let handler =
12141            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12142                .with_supervisor(supervisor);
12143
12144        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12145        hello_via_sink(
12146            &handler,
12147            &target_ctx,
12148            &mut target_rx,
12149            hello_frame("target", PROTOCOL_VERSION, 1),
12150        )
12151        .await;
12152
12153        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12154        let route_handler = handler.clone();
12155        let route_task = tokio::spawn(async move {
12156            route_handler
12157                .handle_control_frame(
12158                    &client_ctx,
12159                    route_open_frame_with_admission_facts(
12160                        30,
12161                        "target",
12162                        unique_project_root("admission-facts"),
12163                        Some(subc_control::ConsumerIdentity {
12164                            module_id: "fed".to_string(),
12165                            launch_nonce: "fed-nonce".to_string(),
12166                        }),
12167                        None,
12168                    ),
12169                )
12170                .await
12171                .unwrap()
12172        });
12173
12174        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
12175        // the very mutation it exists to catch -- a guard that admits nobody -- no
12176        // bind is ever sent, so it HUNG rather than failing. That reproduces the
12177        // exact defect being fixed: a total revocation detected only as a stalled
12178        // suite, which reads as flakiness and invites a retry. An acceptance test
12179        // that waits for an effect must bound the wait, or a red becomes a hang.
12180        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12181            .await
12182            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12183            .expect("module control channel closed before route.bind");
12184        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12185        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12186            panic!("expected route.bind")
12187        };
12188        assert_eq!(
12189            principal,
12190            Some(Principal::Reserved {
12191                module_id: "fed".to_string()
12192            }),
12193            "a correctly attested consumer must be stamped Reserved for its own id"
12194        );
12195
12196        handler
12197            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12198            .await
12199            .unwrap();
12200        assert!(route_task.await.unwrap().is_empty());
12201        assert!(
12202            matches!(
12203                serde_json::from_slice::<ClientControlResponse>(
12204                    &client_rx.recv().await.unwrap().body
12205                )
12206                .unwrap(),
12207                ClientControlResponse::RouteOpen { .. }
12208            ),
12209            "the route must actually open, not merely avoid an error"
12210        );
12211    }
12212
12213    #[tokio::test(start_paused = true)]
12214    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12215        let registry = Arc::new(Registry::default());
12216        let forwarding = Arc::new(ForwardingTable::default());
12217        let supervisor = SupervisorHandle::new();
12218        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12219        let handler =
12220            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12221                .with_supervisor(supervisor);
12222
12223        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12224        hello_via_sink(
12225            &handler,
12226            &target_ctx,
12227            &mut target_rx,
12228            hello_frame("target", PROTOCOL_VERSION, 1),
12229        )
12230        .await;
12231
12232        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12233        let direct_handler = handler.clone();
12234        let direct_open = tokio::spawn(async move {
12235            direct_handler
12236                .handle_control_frame(
12237                    &direct_ctx,
12238                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
12239                )
12240                .await
12241                .unwrap()
12242        });
12243        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12244            .await
12245            .expect("no direct route.bind within 5s")
12246            .expect("target control channel closed before direct route.bind");
12247        handler
12248            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12249            .await
12250            .unwrap();
12251        assert!(direct_open.await.unwrap().is_empty());
12252        let _ = direct_rx.recv().await.unwrap();
12253
12254        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12255        let reserved_handler = handler.clone();
12256        let reserved_open = tokio::spawn(async move {
12257            reserved_handler
12258                .handle_control_frame(
12259                    &reserved_ctx,
12260                    route_open_frame_with_admission_facts(
12261                        3,
12262                        "target",
12263                        unique_project_root("admission-facts"),
12264                        Some(ConsumerIdentity {
12265                            module_id: "fed".to_string(),
12266                            launch_nonce: "fed-nonce".to_string(),
12267                        }),
12268                        None,
12269                    ),
12270                )
12271                .await
12272                .unwrap()
12273        });
12274        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12275            .await
12276            .expect("no reserved route.bind within 5s")
12277            .expect("target control channel closed before reserved route.bind");
12278        handler
12279            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12280            .await
12281            .unwrap();
12282        assert!(reserved_open.await.unwrap().is_empty());
12283        let _ = reserved_rx.recv().await.unwrap();
12284
12285        forwarding
12286            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12287            .unwrap();
12288        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12289        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12290            module_id: Some("target".to_string()),
12291        })
12292        .unwrap();
12293        let census_frame =
12294            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12295        let response = handler
12296            .handle_control_frame(&census_ctx, census_frame)
12297            .await
12298            .unwrap()
12299            .pop()
12300            .unwrap();
12301        let actual: Value = serde_json::from_slice(&response.body).unwrap();
12302        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12303        assert!(matches!(
12304            decoded,
12305            ClientControlResponse::SupervisorRoutes { .. }
12306        ));
12307        let routes = actual["modules"][0]["routes"].as_array().unwrap();
12308        assert_eq!(routes.len(), 2);
12309        assert!(routes.iter().all(|route| route["draining"] == true));
12310        // The census carries WHY: the reason the drain was begun with, in the
12311        // route.closing vocabulary, on every draining route this drain marked.
12312        assert!(
12313            routes.iter().all(|route| route["drain_reason"] == "reload"),
12314            "draining routes must name the drain's reason: {routes:?}"
12315        );
12316        assert!(routes.iter().any(|route| {
12317            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12318        }));
12319        assert!(routes.iter().any(|route| {
12320            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12321        }));
12322
12323        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12324            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12325        if std::env::var_os("UPDATE_GOLDEN").is_some() {
12326            std::fs::write(
12327                &golden_path,
12328                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12329            )
12330            .unwrap();
12331        }
12332        let expected: Value =
12333            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12334        assert_eq!(actual, expected);
12335    }
12336
12337    async fn query_live_roots(
12338        handler: &ControlHandler,
12339        module_ctx: &RouteCtx,
12340    ) -> ModuleControlResponseToModule {
12341        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12342        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12343        let response = handler
12344            .handle_control_frame(module_ctx, frame)
12345            .await
12346            .unwrap()
12347            .pop()
12348            .unwrap();
12349        serde_json::from_slice(&response.body).unwrap()
12350    }
12351
12352    #[tokio::test(start_paused = true)]
12353    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12354        let registry = Arc::new(Registry::default());
12355        let forwarding = Arc::new(ForwardingTable::default());
12356        let handler = ControlHandler::with_forwarding(registry, forwarding);
12357        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12358        hello_via_sink(
12359            &handler,
12360            &target_ctx,
12361            &mut target_rx,
12362            hello_frame("target", PROTOCOL_VERSION, 1),
12363        )
12364        .await;
12365        let root = unique_project_root("live-roots-known");
12366        let path = ProjectRootId::from_path_allowing_missing(root.path())
12367            .unwrap()
12368            .as_path()
12369            .to_path_buf();
12370        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12371        let open_handler = handler.clone();
12372        let opened = tokio::spawn(async move {
12373            open_handler
12374                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12375                .await
12376                .unwrap()
12377        });
12378        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12379            .await
12380            .unwrap()
12381            .unwrap();
12382        handler
12383            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12384            .await
12385            .unwrap();
12386        assert!(opened.await.unwrap().is_empty());
12387        let _ = client_rx.recv().await.unwrap();
12388
12389        let root = unique_project_root("live-roots-pending");
12390        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12391            .unwrap()
12392            .as_path()
12393            .to_path_buf();
12394        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12395        let open_handler = handler.clone();
12396        let pending = tokio::spawn(async move {
12397            open_handler
12398                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12399                .await
12400                .unwrap()
12401        });
12402        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12403            .await
12404            .unwrap()
12405            .unwrap();
12406        let actual = query_live_roots(&handler, &target_ctx).await;
12407        let ModuleControlResponseToModule::LiveRoots {
12408            roots,
12409            unknown_root_bindings,
12410            total_bindings,
12411        } = actual
12412        else {
12413            panic!("expected live roots")
12414        };
12415        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12416        assert_eq!(unknown_root_bindings, 0);
12417        assert_eq!(
12418            roots.len(),
12419            2,
12420            "root-known arm must retain each canonical root"
12421        );
12422        assert_eq!(
12423            total_bindings,
12424            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12425        );
12426        let counts = roots
12427            .iter()
12428            .map(|root| (root.project_root.clone(), root.bound, root.pending))
12429            .collect::<Vec<_>>();
12430        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12431        expected.sort_by(|a, b| a.0.cmp(&b.0));
12432        assert_eq!(
12433            counts, expected,
12434            "roots must sort by path and count pending separately"
12435        );
12436        handler
12437            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12438            .await
12439            .unwrap();
12440        assert!(pending.await.unwrap().is_empty());
12441    }
12442
12443    #[tokio::test(start_paused = true)]
12444    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12445        let forwarding = Arc::new(ForwardingTable::default());
12446        let handler =
12447            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12448        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12449        hello_via_sink(
12450            &handler,
12451            &target_ctx,
12452            &mut target_rx,
12453            hello_frame("target", PROTOCOL_VERSION, 1),
12454        )
12455        .await;
12456        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12457        let pending = forwarding
12458            .begin_route_bind_relay_for_test(
12459                client_ctx.connection_id,
12460                client_ctx.egress.clone(),
12461                2,
12462                "target",
12463            )
12464            .unwrap();
12465        forwarding
12466            .complete_pending_relay(
12467                target_ctx.connection_id,
12468                pending.corr,
12469                RouteBindRelayOutcome::Accepted,
12470            )
12471            .unwrap();
12472        let actual = query_live_roots(&handler, &target_ctx).await;
12473        let ModuleControlResponseToModule::LiveRoots {
12474            roots,
12475            unknown_root_bindings,
12476            total_bindings,
12477        } = actual
12478        else {
12479            panic!("expected live roots")
12480        };
12481        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12482        assert_eq!(
12483            unknown_root_bindings, 1,
12484            "unknown-root arm must not read as no bindings"
12485        );
12486        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12487        assert_eq!(
12488            total_bindings,
12489            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12490        );
12491    }
12492
12493    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12494    /// so the ack has to be on its outbound queue before the module is
12495    /// routable. The connection loop writes a handler's replies only after the
12496    /// handler returns; this test stops in exactly that gap, runs a real
12497    /// route.open from another connection, and only then writes whatever the
12498    /// HELLO handler returned, the way the loop would. If the ack were still a
12499    /// reply, the route.bind request would reach the module first.
12500    #[tokio::test(start_paused = true)]
12501    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12502        let forwarding = Arc::new(ForwardingTable::default());
12503        let handler =
12504            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12505        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12506        let replies = handler
12507            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12508            .await
12509            .unwrap();
12510        let queued_by_hello = module_rx.len();
12511
12512        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12513        let open_handler = handler.clone();
12514        let open = tokio::spawn(async move {
12515            open_handler
12516                .handle_control_frame(
12517                    &client_ctx,
12518                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12519                )
12520                .await
12521                .unwrap()
12522        });
12523        // Let the route.open run until its route.bind is on the module's queue.
12524        let mut spins = 0;
12525        while module_rx.len() == queued_by_hello {
12526            spins += 1;
12527            assert!(spins < 10_000, "route.open never queued a route.bind");
12528            tokio::task::yield_now().await;
12529        }
12530
12531        // Now the connection loop's half: write the HELLO handler's replies.
12532        for reply in replies {
12533            module_ctx.egress.send(reply).await.unwrap();
12534        }
12535
12536        let first = module_rx.recv().await.unwrap().frame;
12537        assert_eq!(
12538            first.header.ty,
12539            FrameType::HelloAck,
12540            "the first frame a registering module reads must be its HELLO_ACK"
12541        );
12542        assert_eq!(first.header.corr, 7);
12543        let second = module_rx.recv().await.unwrap().frame;
12544        assert_eq!(second.header.ty, FrameType::Request);
12545        assert!(
12546            matches!(
12547                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12548                ModuleControlRequest::RouteBind { .. }
12549            ),
12550            "the route.bind follows the ack"
12551        );
12552        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12553
12554        handler
12555            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12556            .await
12557            .unwrap();
12558        assert!(open.await.unwrap().is_empty());
12559        let _ = client_rx.recv().await.unwrap();
12560    }
12561
12562    #[tokio::test(start_paused = true)]
12563    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12564        let handler = ControlHandler::with_forwarding(
12565            Arc::new(Registry::default()),
12566            Arc::new(ForwardingTable::default()),
12567        );
12568        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12569        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12570        hello_via_sink(
12571            &handler,
12572            &first_ctx,
12573            &mut first_rx,
12574            hello_frame("first", PROTOCOL_VERSION, 1),
12575        )
12576        .await;
12577        hello_via_sink(
12578            &handler,
12579            &second_ctx,
12580            &mut second_rx,
12581            hello_frame("second", PROTOCOL_VERSION, 2),
12582        )
12583        .await;
12584        let root = unique_project_root("second-only");
12585        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12586        let cloned = handler.clone();
12587        let open = tokio::spawn(async move {
12588            cloned
12589                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12590                .await
12591                .unwrap()
12592        });
12593        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12594            .await
12595            .unwrap()
12596            .unwrap();
12597        let first = query_live_roots(&handler, &first_ctx).await;
12598        let second = query_live_roots(&handler, &second_ctx).await;
12599        assert!(
12600            matches!(
12601                first,
12602                ModuleControlResponseToModule::LiveRoots {
12603                    total_bindings: 0,
12604                    ..
12605                }
12606            ),
12607            "cross-module scope must not expose another module's roots"
12608        );
12609        assert!(
12610            matches!(
12611                second,
12612                ModuleControlResponseToModule::LiveRoots {
12613                    total_bindings: 1,
12614                    ..
12615                }
12616            ),
12617            "second module must see its pending route"
12618        );
12619        handler
12620            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12621            .await
12622            .unwrap();
12623        assert!(open.await.unwrap().is_empty());
12624    }
12625
12626    #[tokio::test(start_paused = true)]
12627    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12628        let handler = ControlHandler::with_forwarding(
12629            Arc::new(Registry::default()),
12630            Arc::new(ForwardingTable::default()),
12631        );
12632        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12633        hello_via_sink(
12634            &handler,
12635            &target_ctx,
12636            &mut target_rx,
12637            hello_frame("target", PROTOCOL_VERSION, 1),
12638        )
12639        .await;
12640        let actual = query_live_roots(&handler, &target_ctx).await;
12641        let ModuleControlResponseToModule::LiveRoots {
12642            roots,
12643            unknown_root_bindings,
12644            total_bindings,
12645        } = actual
12646        else {
12647            panic!("expected live roots")
12648        };
12649        assert!(roots.is_empty());
12650        assert_eq!(unknown_root_bindings, 0);
12651        assert_eq!(total_bindings, 0);
12652        assert_eq!(
12653            total_bindings,
12654            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12655        );
12656    }
12657
12658    /// Read the vendored fed corpus rather than hand-building a package.
12659    ///
12660    /// A hand-built object encodes what the test author believed the carrier
12661    /// emits. These vectors are what it actually emits, and one of them exists
12662    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
12663    /// ignores additive unknown fields at the traversal emit terminus."
12664    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12665        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12666            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12667        let text = std::fs::read_to_string(&path)
12668            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12669        let vectors: Vec<(String, Value)> = text
12670            .lines()
12671            .filter(|line| !line.trim().is_empty())
12672            .map(|line| {
12673                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12674                let id = entry["corpus_id"]
12675                    .as_str()
12676                    .expect("every vector carries a corpus_id")
12677                    .to_string();
12678                (id, entry["package"].clone())
12679            })
12680            .collect();
12681        // Pin the count: a corpus that silently shrinks would take its coverage
12682        // with it, and a suite reading N-1 vectors reports the same clean pass
12683        // as one reading N.
12684        assert_eq!(
12685            vectors.len(),
12686            3,
12687            "vendored fed corpus changed size; re-sync from subc-federation"
12688        );
12689
12690        // Pin what makes the corpus DISCRIMINATING, not just present.
12691        //
12692        // The relay test below takes its expected value from the corpus, so the
12693        // corpus supplies the test's power to detect a lossy relay rather than
12694        // its correctness. A relay that dropped unrecognised fields would still
12695        // be caught -- but only by a package carrying fields it does not know.
12696        // Shrink every package to the handful of keys any implementation would
12697        // recognise and the test keeps passing over an input that can no longer
12698        // fail, which is the same clean green as a corpus that shrank away.
12699        //
12700        // So assert the precondition rather than duplicating the packages here:
12701        // at least one vector must carry a field beyond the small common set.
12702        // That is one claim to maintain instead of nine, and it fails loudly if
12703        // a re-sync ever flattens the corpus.
12704        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12705        let richest = vectors
12706            .iter()
12707            .filter_map(|(_, package)| package.as_object())
12708            .map(|object| {
12709                object
12710                    .keys()
12711                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12712                    .count()
12713            })
12714            .max()
12715            .unwrap_or(0);
12716        assert!(
12717            richest >= 2,
12718            "vendored corpus no longer carries a package with unmodelled fields, \
12719             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12720        );
12721
12722        vectors
12723    }
12724
12725    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12726    ///
12727    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12728    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12729    /// three-key object, so a relay that quietly dropped fields it did not
12730    /// recognise would satisfy it. These vectors carry nine keys including ones
12731    /// this crate has no type for, so a typed relay fails here and only here.
12732    #[tokio::test]
12733    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12734        for (corpus_id, package) in fed_admission_facts_vectors() {
12735            let registry = Arc::new(Registry::default());
12736            let forwarding = Arc::new(ForwardingTable::default());
12737            let supervisor = SupervisorHandle::new();
12738            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12739            let handler =
12740                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12741                    .with_supervisor(supervisor)
12742                    .with_admission_facts_config(
12743                        Some("fed".to_string()),
12744                        Some(vec!["target".to_string()]),
12745                    );
12746
12747            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12748            hello_via_sink(
12749                &handler,
12750                &target_ctx,
12751                &mut target_rx,
12752                hello_frame("target", PROTOCOL_VERSION, 1),
12753            )
12754            .await;
12755
12756            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12757            let route_handler = handler.clone();
12758            let expected = package.clone();
12759            let route_task = tokio::spawn(async move {
12760                route_handler
12761                    .handle_control_frame(
12762                        &client_ctx,
12763                        route_open_frame_with_admission_facts(
12764                            20,
12765                            "target",
12766                            unique_project_root("admission-facts"),
12767                            Some(subc_control::ConsumerIdentity {
12768                                module_id: "fed".to_string(),
12769                                launch_nonce: "fed-nonce".to_string(),
12770                            }),
12771                            Some(package),
12772                        ),
12773                    )
12774                    .await
12775                    .unwrap()
12776            });
12777
12778            let bind_frame = target_rx.recv().await.unwrap();
12779            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12780            let ModuleControlRequest::RouteBind {
12781                admission_facts, ..
12782            } = bind
12783            else {
12784                panic!("{corpus_id}: expected route.bind")
12785            };
12786            assert_eq!(
12787                admission_facts,
12788                Some(expected),
12789                "{corpus_id}: relay must not add, drop or reshape any field"
12790            );
12791
12792            handler
12793                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12794                .await
12795                .unwrap();
12796            route_task.await.unwrap();
12797        }
12798    }
12799
12800    #[tokio::test]
12801    async fn route_bind_never_relays_the_openers_build_class() {
12802        let registry = Arc::new(Registry::default());
12803        let forwarding = Arc::new(ForwardingTable::default());
12804        let handler =
12805            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12806        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12807        hello_via_sink(
12808            &handler,
12809            &target_ctx,
12810            &mut target_rx,
12811            hello_frame("target", PROTOCOL_VERSION, 1),
12812        )
12813        .await;
12814
12815        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
12816        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12817        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12818            busy_refusals: None,
12819            target: RouteTarget::ToolProvider {
12820                module_id: "target".to_string(),
12821            },
12822            identity,
12823            consumer_identity: None,
12824            consumer_capabilities: None,
12825            role_versions: None,
12826            admission_facts: None,
12827            scope: None,
12828        })
12829        .unwrap();
12830        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12831
12832        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12833        let route_handler = handler.clone();
12834        let route_task = tokio::spawn(async move {
12835            route_handler
12836                .handle_control_frame(&client_ctx, open)
12837                .await
12838                .unwrap()
12839        });
12840        let bind_frame = target_rx.recv().await.unwrap();
12841        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12842        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12843            panic!("expected route.bind")
12844        };
12845        assert_eq!(
12846            identity.build_class, None,
12847            "route.bind carried the build class"
12848        );
12849        assert!(
12850            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12851            "the route.bind bytes still name build_class"
12852        );
12853        handler
12854            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12855            .await
12856            .unwrap();
12857        route_task.await.unwrap();
12858    }
12859
12860    #[tokio::test]
12861    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12862        let registry = Arc::new(Registry::default());
12863        let forwarding = Arc::new(ForwardingTable::default());
12864        let supervisor = SupervisorHandle::new();
12865        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12866        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12867        let handler =
12868            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12869                .with_supervisor(supervisor)
12870                .with_admission_facts_config(
12871                    Some("fed".to_string()),
12872                    Some(vec!["target".to_string()]),
12873                );
12874
12875        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12876        hello_via_sink(
12877            &handler,
12878            &target_ctx,
12879            &mut target_rx,
12880            hello_frame("target", PROTOCOL_VERSION, 1),
12881        )
12882        .await;
12883        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12884        hello_via_sink(
12885            &handler,
12886            &other_ctx,
12887            &mut other_rx,
12888            hello_frame("other", PROTOCOL_VERSION, 2),
12889        )
12890        .await;
12891
12892        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12893        let expected_facts = facts.clone();
12894        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12895        let route_handler = handler.clone();
12896        let route_task = tokio::spawn(async move {
12897            route_handler
12898                .handle_control_frame(
12899                    &client_ctx,
12900                    route_open_frame_with_admission_facts(
12901                        10,
12902                        "target",
12903                        unique_project_root("admission-facts"),
12904                        Some(subc_control::ConsumerIdentity {
12905                            module_id: "fed".to_string(),
12906                            launch_nonce: "fed-nonce".to_string(),
12907                        }),
12908                        Some(facts.clone()),
12909                    ),
12910                )
12911                .await
12912                .unwrap()
12913        });
12914        let bind_frame = target_rx.recv().await.unwrap();
12915        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12916        let ModuleControlRequest::RouteBind {
12917            admission_facts, ..
12918        } = bind
12919        else {
12920            panic!("expected route.bind")
12921        };
12922        assert_eq!(admission_facts, Some(expected_facts));
12923        handler
12924            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12925            .await
12926            .unwrap();
12927        assert!(route_task.await.unwrap().is_empty());
12928        assert!(matches!(
12929            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12930                .unwrap(),
12931            ClientControlResponse::RouteOpen { .. }
12932        ));
12933
12934        let direct = handler
12935            .handle_control_frame(
12936                &route_ctx(ConnectionId::new(73)).0,
12937                route_open_frame_with_admission_facts(
12938                    11,
12939                    "target",
12940                    unique_project_root("admission-facts"),
12941                    None,
12942                    Some(json!({"x": 1})),
12943                ),
12944            )
12945            .await
12946            .unwrap();
12947        assert_eq!(
12948            parse_error(&direct[0])["code"],
12949            "admission_facts_not_permitted"
12950        );
12951
12952        let different_reserved = handler
12953            .handle_control_frame(
12954                &route_ctx(ConnectionId::new(77)).0,
12955                route_open_frame_with_admission_facts(
12956                    15,
12957                    "target",
12958                    unique_project_root("admission-facts"),
12959                    Some(subc_control::ConsumerIdentity {
12960                        module_id: "other".to_string(),
12961                        launch_nonce: "other-nonce".to_string(),
12962                    }),
12963                    Some(json!({"x": 1})),
12964                ),
12965            )
12966            .await
12967            .unwrap();
12968        assert_eq!(
12969            parse_error(&different_reserved[0])["code"],
12970            "admission_facts_not_permitted"
12971        );
12972
12973        let other_target = handler
12974            .handle_control_frame(
12975                &route_ctx(ConnectionId::new(74)).0,
12976                route_open_frame_with_admission_facts(
12977                    12,
12978                    "other",
12979                    unique_project_root("admission-facts"),
12980                    Some(subc_control::ConsumerIdentity {
12981                        module_id: "fed".to_string(),
12982                        launch_nonce: "fed-nonce".to_string(),
12983                    }),
12984                    Some(json!({"x": 1})),
12985                ),
12986            )
12987            .await
12988            .unwrap();
12989        assert_eq!(
12990            parse_error(&other_target[0])["code"],
12991            "admission_facts_target_not_allowed"
12992        );
12993
12994        let nonexistent = handler
12995            .handle_control_frame(
12996                &route_ctx(ConnectionId::new(75)).0,
12997                route_open_frame_with_admission_facts(
12998                    13,
12999                    "missing",
13000                    unique_project_root("admission-facts"),
13001                    None,
13002                    Some(json!({"x": 1})),
13003                ),
13004            )
13005            .await
13006            .unwrap();
13007        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13008
13009        let described = handler
13010            .handle_control_frame(
13011                &route_ctx(ConnectionId::new(76)).0,
13012                Frame::build(
13013                    FrameType::Request,
13014                    control_flags(),
13015                    0,
13016                    0,
13017                    14,
13018                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13019                )
13020                .unwrap(),
13021            )
13022            .await
13023            .unwrap();
13024        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13025            serde_json::from_slice(&described[0].body).unwrap()
13026        else {
13027            panic!("expected server.describe response")
13028        };
13029        assert!(capabilities
13030            .iter()
13031            .any(|cap| cap == "admission_facts_relay_v1"));
13032    }
13033
13034    #[tokio::test]
13035    async fn admission_facts_without_configured_carrier_are_rejected() {
13036        let registry = Arc::new(Registry::default());
13037        let forwarding = Arc::new(ForwardingTable::default());
13038        let handler = ControlHandler::with_forwarding(registry, forwarding);
13039        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13040        hello_via_sink(
13041            &handler,
13042            &target_ctx,
13043            &mut target_rx,
13044            hello_frame("target", PROTOCOL_VERSION, 1),
13045        )
13046        .await;
13047
13048        let responses = handler
13049            .handle_control_frame(
13050                &route_ctx(ConnectionId::new(79)).0,
13051                route_open_frame_with_admission_facts(
13052                    16,
13053                    "target",
13054                    unique_project_root("admission-facts"),
13055                    None,
13056                    Some(json!({"x": 1})),
13057                ),
13058            )
13059            .await
13060            .unwrap();
13061        assert_eq!(
13062            parse_error(&responses[0])["code"],
13063            "admission_facts_not_permitted"
13064        );
13065    }
13066
13067    #[tokio::test]
13068    async fn route_open_relays_consumer_capabilities_verbatim() {
13069        let registry = Arc::new(Registry::default());
13070        let forwarding = Arc::new(ForwardingTable::default());
13071        let handler =
13072            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13073        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13074        hello_via_sink(
13075            &handler,
13076            &module_ctx,
13077            &mut module_rx,
13078            hello_frame("aft", PROTOCOL_VERSION, 7),
13079        )
13080        .await;
13081
13082        let expected = vec!["elicitation".to_string(), "roots".to_string()];
13083        let expected_for_request = expected.clone();
13084        let project_root = unique_project_root("consumer-capabilities-present");
13085        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13086        let route_handler = handler.clone();
13087        let route_task = tokio::spawn(async move {
13088            route_handler
13089                .handle_control_frame(
13090                    &client_ctx,
13091                    route_open_frame_with_consumer_capabilities(
13092                        401,
13093                        "aft",
13094                        project_root,
13095                        Some(expected_for_request),
13096                    ),
13097                )
13098                .await
13099                .unwrap()
13100        });
13101        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13102            .await
13103            .unwrap()
13104            .unwrap();
13105        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13106        let ModuleControlRequest::RouteBind {
13107            consumer_capabilities,
13108            ..
13109        } = bind
13110        else {
13111            panic!("expected route.bind request, got {bind:?}");
13112        };
13113        assert_eq!(consumer_capabilities, Some(expected.clone()));
13114
13115        handler
13116            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13117            .await
13118            .unwrap();
13119        let route_response = route_task.await.unwrap();
13120        assert!(route_response.is_empty());
13121        let published = client_rx.recv().await.unwrap();
13122        assert!(matches!(
13123            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13124            ClientControlResponse::RouteOpen { .. }
13125        ));
13126    }
13127
13128    #[tokio::test]
13129    async fn route_open_without_consumer_capabilities_relays_none() {
13130        let registry = Arc::new(Registry::default());
13131        let forwarding = Arc::new(ForwardingTable::default());
13132        let handler =
13133            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13134        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13135        hello_via_sink(
13136            &handler,
13137            &module_ctx,
13138            &mut module_rx,
13139            hello_frame("aft", PROTOCOL_VERSION, 7),
13140        )
13141        .await;
13142
13143        let project_root = unique_project_root("consumer-capabilities-absent");
13144        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13145        let route_handler = handler.clone();
13146        let route_task = tokio::spawn(async move {
13147            route_handler
13148                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13149                .await
13150                .unwrap()
13151        });
13152        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13153            .await
13154            .unwrap()
13155            .unwrap();
13156        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13157        let ModuleControlRequest::RouteBind {
13158            consumer_capabilities,
13159            ..
13160        } = bind
13161        else {
13162            panic!("expected route.bind request, got {bind:?}");
13163        };
13164        assert_eq!(consumer_capabilities, None);
13165
13166        handler
13167            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13168            .await
13169            .unwrap();
13170        let route_response = route_task.await.unwrap();
13171        assert!(route_response.is_empty());
13172        let published = client_rx.recv().await.unwrap();
13173        assert!(matches!(
13174            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13175            ClientControlResponse::RouteOpen { .. }
13176        ));
13177    }
13178
13179    /// Opens a route to a freshly registered `aft` with `sent` as the
13180    /// route.open's role_versions, acks the bind, and returns the role_versions
13181    /// the module's bind carried.
13182    async fn bind_role_versions_for(
13183        sent: Option<BTreeMap<String, String>>,
13184        connection: u64,
13185    ) -> Option<BTreeMap<String, String>> {
13186        let registry = Arc::new(Registry::default());
13187        let forwarding = Arc::new(ForwardingTable::default());
13188        let handler =
13189            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13190        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13191        hello_via_sink(
13192            &handler,
13193            &module_ctx,
13194            &mut module_rx,
13195            hello_frame("aft", PROTOCOL_VERSION, 7),
13196        )
13197        .await;
13198        let project_root = unique_project_root("role-versions");
13199        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13200        let route_handler = handler.clone();
13201        let route_task = tokio::spawn(async move {
13202            route_handler
13203                .handle_control_frame(
13204                    &client_ctx,
13205                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
13206                )
13207                .await
13208                .unwrap()
13209        });
13210        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13211            .await
13212            .expect("a well-formed route.open reaches the module as a bind")
13213            .unwrap();
13214        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13215        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13216            panic!("expected route.bind request, got {bind:?}");
13217        };
13218        handler
13219            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13220            .await
13221            .unwrap();
13222        assert!(route_task.await.unwrap().is_empty());
13223        let published = client_rx.recv().await.unwrap();
13224        assert!(matches!(
13225            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13226            ClientControlResponse::RouteOpen { .. }
13227        ));
13228        role_versions
13229    }
13230
13231    #[tokio::test]
13232    async fn route_open_relays_role_versions_verbatim() {
13233        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13234        assert_eq!(
13235            bind_role_versions_for(Some(sent.clone()), 141).await,
13236            Some(sent)
13237        );
13238    }
13239
13240    /// An empty map declares nothing, so the provider sees no field rather
13241    /// than an empty object it would have to treat as a second "none".
13242    #[tokio::test]
13243    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13244        assert_eq!(bind_role_versions_for(None, 143).await, None);
13245        assert_eq!(
13246            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13247            None
13248        );
13249    }
13250
13251    #[tokio::test]
13252    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13253        let registry = Arc::new(Registry::default());
13254        let forwarding = Arc::new(ForwardingTable::default());
13255        let handler =
13256            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13257        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13258        hello_via_sink(
13259            &handler,
13260            &module_ctx,
13261            &mut module_rx,
13262            hello_frame("aft", PROTOCOL_VERSION, 7),
13263        )
13264        .await;
13265
13266        let nine: BTreeMap<String, String> = (0..9)
13267            .map(|index| (format!("role-{index}"), "v1".to_string()))
13268            .collect();
13269        for (label, malformed) in [
13270            (
13271                "invalid role name",
13272                role_versions(&[("Tool_Provider", "v1")]),
13273            ),
13274            ("invalid version", role_versions(&[("tool-provider", "v0")])),
13275            ("nine entries", nine),
13276        ] {
13277            // A refused open answers at once; one that reached the module
13278            // would wait for its bind ack and trip this timeout.
13279            let responses = tokio::time::timeout(
13280                Duration::from_secs(1),
13281                handler.handle_control_frame(
13282                    &route_ctx(ConnectionId::new(148)).0,
13283                    route_open_frame_with_role_versions(
13284                        404,
13285                        "aft",
13286                        unique_project_root("role-versions-malformed"),
13287                        Some(malformed),
13288                    ),
13289                ),
13290            )
13291            .await
13292            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13293            .unwrap();
13294            assert_eq!(responses.len(), 1, "{label}");
13295            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13296            let error = parse_error(&responses[0]);
13297            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13298            assert_eq!(
13299                error["detail"]["field"], "role_versions",
13300                "{label}: {error}"
13301            );
13302            assert!(
13303                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13304                "{label}: a malformed declaration is terminal"
13305            );
13306            assert!(
13307                module_rx.try_recv().is_err(),
13308                "{label}: the module must never see a bind"
13309            );
13310        }
13311    }
13312
13313    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
13314    /// consumer can tell this daemon forwards the field from one that would
13315    /// drop it.
13316    #[tokio::test]
13317    async fn route_role_versions_capability_is_advertised() {
13318        let handler = ControlHandler::new(Arc::new(Registry::default()));
13319        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13320        let ack = hello_via_sink(
13321            &handler,
13322            &ctx,
13323            &mut rx,
13324            hello_frame("m", PROTOCOL_VERSION, 1),
13325        )
13326        .await;
13327        let ack = parse_ack(&ack);
13328        assert!(
13329            ack.subc_capabilities
13330                .iter()
13331                .any(|c| c == "route-role-versions/v1"),
13332            "{:?}",
13333            ack.subc_capabilities
13334        );
13335
13336        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13337        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13338        let reply = handler
13339            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13340            .await
13341            .unwrap()
13342            .pop()
13343            .unwrap();
13344        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13345            serde_json::from_slice(&reply.body).unwrap()
13346        else {
13347            panic!("not a server.describe reply");
13348        };
13349        assert!(
13350            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13351            "{capabilities:?}"
13352        );
13353    }
13354
13355    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
13356    /// catalog entry without `events` (the module declares none) from a daemon
13357    /// that never reports declared events.
13358    #[tokio::test]
13359    async fn catalog_events_capability_is_advertised() {
13360        let handler = ControlHandler::new(Arc::new(Registry::default()));
13361        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13362        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13363        let reply = handler
13364            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13365            .await
13366            .unwrap()
13367            .pop()
13368            .unwrap();
13369        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13370            serde_json::from_slice(&reply.body).unwrap()
13371        else {
13372            panic!("not a server.describe reply");
13373        };
13374        assert!(
13375            capabilities
13376                .iter()
13377                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13378            "{capabilities:?}"
13379        );
13380    }
13381
13382    #[tokio::test]
13383    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13384        let handler = ControlHandler::new(Arc::new(Registry::default()));
13385        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13386        let ack = hello_via_sink(
13387            &handler,
13388            &module,
13389            &mut module_rx,
13390            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13391        )
13392        .await;
13393        let ack = parse_ack(&ack);
13394        assert!(
13395            ack.subc_capabilities
13396                .iter()
13397                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13398            "{:?}",
13399            ack.subc_capabilities
13400        );
13401        assert!(ack
13402            .subc_capabilities
13403            .iter()
13404            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13405
13406        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13407        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13408        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13409        let reply = handler
13410            .handle_control_frame(&client, frame)
13411            .await
13412            .unwrap()
13413            .pop()
13414            .unwrap();
13415        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13416            serde_json::from_slice(&reply.body).unwrap()
13417        else {
13418            panic!("not a server.describe reply");
13419        };
13420        assert!(
13421            capabilities
13422                .iter()
13423                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13424            "{capabilities:?}"
13425        );
13426        assert!(capabilities
13427            .iter()
13428            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13429    }
13430
13431    #[tokio::test]
13432    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13433        let handler = ControlHandler::new(Arc::new(Registry::default()));
13434        let (module, mut rx) = route_ctx(ConnectionId::new(156));
13435        let ack = hello_via_sink(
13436            &handler,
13437            &module,
13438            &mut rx,
13439            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13440        )
13441        .await;
13442        assert!(parse_ack(&ack)
13443            .subc_capabilities
13444            .iter()
13445            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13446        assert!(parse_ack(&ack)
13447            .subc_capabilities
13448            .iter()
13449            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13450        let (client, _rx) = route_ctx(ConnectionId::new(157));
13451        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13452        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13453        let reply = handler
13454            .handle_control_frame(&client, frame)
13455            .await
13456            .unwrap()
13457            .pop()
13458            .unwrap();
13459        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13460            serde_json::from_slice(&reply.body).unwrap()
13461        else {
13462            panic!("server.describe reply")
13463        };
13464        assert!(capabilities
13465            .iter()
13466            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13467        assert!(capabilities
13468            .iter()
13469            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13470    }
13471
13472    #[tokio::test]
13473    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13474        let handler = ControlHandler::new(Arc::new(Registry::default()));
13475        let (module, mut rx) = route_ctx(ConnectionId::new(158));
13476        let ack = hello_via_sink(
13477            &handler,
13478            &module,
13479            &mut rx,
13480            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13481        )
13482        .await;
13483        assert!(parse_ack(&ack)
13484            .subc_capabilities
13485            .iter()
13486            .any(|cap| cap == "scope-workdir/v1"));
13487        let (client, _rx) = route_ctx(ConnectionId::new(159));
13488        let frame = Frame::build(
13489            FrameType::Request,
13490            control_flags(),
13491            0,
13492            0,
13493            5,
13494            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13495        )
13496        .unwrap();
13497        let reply = handler
13498            .handle_control_frame(&client, frame)
13499            .await
13500            .unwrap()
13501            .remove(0);
13502        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13503            serde_json::from_slice(&reply.body).unwrap()
13504        else {
13505            panic!("server.describe reply")
13506        };
13507        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13508    }
13509
13510    #[tokio::test]
13511    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13512        let handler = ControlHandler::new(Arc::new(Registry::default()));
13513        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13514        let ack = hello_via_sink(
13515            &handler,
13516            &module,
13517            &mut rx,
13518            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13519        )
13520        .await;
13521        assert!(parse_ack(&ack)
13522            .subc_capabilities
13523            .iter()
13524            .any(|cap| cap == "worker-runs/v1"));
13525        let (client, _rx) = route_ctx(ConnectionId::new(161));
13526        let frame = Frame::build(
13527            FrameType::Request,
13528            control_flags(),
13529            0,
13530            0,
13531            5,
13532            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13533        )
13534        .unwrap();
13535        let reply = handler
13536            .handle_control_frame(&client, frame)
13537            .await
13538            .unwrap()
13539            .remove(0);
13540        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13541            serde_json::from_slice(&reply.body).unwrap()
13542        else {
13543            panic!("server.describe reply")
13544        };
13545        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13546    }
13547
13548    #[tokio::test]
13549    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13550        let handler = ControlHandler::new(Arc::new(Registry::default()));
13551        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13552        let ack = hello_via_sink(
13553            &handler,
13554            &module,
13555            &mut rx,
13556            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13557        )
13558        .await;
13559        assert!(parse_ack(&ack)
13560            .subc_capabilities
13561            .iter()
13562            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13563        let (client, _rx) = route_ctx(ConnectionId::new(155));
13564        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13565        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13566        let reply = handler
13567            .handle_control_frame(&client, frame)
13568            .await
13569            .unwrap()
13570            .pop()
13571            .unwrap();
13572        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13573            serde_json::from_slice(&reply.body).unwrap()
13574        else {
13575            panic!("not a server.describe reply")
13576        };
13577        assert!(capabilities
13578            .iter()
13579            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13580    }
13581
13582    #[tokio::test]
13583    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13584        let registry = Arc::new(Registry::default());
13585        let forwarding = Arc::new(ForwardingTable::default());
13586        let handler =
13587            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13588                .with_health_probe_timeout(Duration::from_secs(5));
13589        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13590        hello_via_sink(
13591            &handler,
13592            &module_ctx,
13593            &mut module_rx,
13594            non_routable_hello_frame_with_control_ops(
13595                "mcp",
13596                300,
13597                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13598            ),
13599        )
13600        .await;
13601        assert!(registry
13602            .get_module("mcp")
13603            .unwrap()
13604            .unwrap()
13605            .manifest
13606            .provides
13607            .is_empty());
13608
13609        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13610        let route_response = handler
13611            .handle_control_frame(
13612                &route_client_ctx,
13613                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13614            )
13615            .await
13616            .unwrap();
13617        assert_eq!(route_response[0].header.ty, FrameType::Error);
13618        assert_eq!(
13619            parse_error(&route_response[0])["code"],
13620            "target_unavailable"
13621        );
13622        assert!(parse_error(&route_response[0])["message"]
13623            .as_str()
13624            .unwrap()
13625            .contains("does not provide the requested target"));
13626        assert!(module_rx.try_recv().is_err());
13627
13628        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13629        let health_handler = handler.clone();
13630        let health_task = tokio::spawn(async move {
13631            health_handler
13632                .handle_control_frame(
13633                    &health_client_ctx,
13634                    supervisor_health_probe_frame(302, "mcp"),
13635                )
13636                .await
13637                .unwrap()
13638        });
13639        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13640            .await
13641            .unwrap()
13642            .unwrap();
13643        assert_eq!(
13644            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13645            ModuleControlRequest::HealthCheck {}
13646        );
13647        handler
13648            .handle_control_frame(
13649                &module_ctx,
13650                health_response(health_frame.header.corr, HealthStatus::Ok),
13651            )
13652            .await
13653            .unwrap();
13654        let health_response = health_task.await.unwrap();
13655        assert_eq!(health_response[0].header.ty, FrameType::Response);
13656        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13657            ClientControlResponse::SupervisorHealthProbe {
13658                module_id, status, ..
13659            } => {
13660                assert_eq!(module_id, "mcp");
13661                assert_eq!(status, HealthStatus::Ok);
13662            }
13663            other => panic!("unexpected health response: {other:?}"),
13664        }
13665
13666        // Exercise the forwarding cleanup path directly while leaving the registry
13667        // advertisement in place. If cleanup leaves a stale control sink behind,
13668        // the next probe will enqueue onto it and wait for the long probe timeout
13669        // instead of returning an immediate no-connection error.
13670        forwarding
13671            .cleanup_connection(module_ctx.connection_id)
13672            .unwrap();
13673        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13674        let cleanup_response = tokio::time::timeout(
13675            Duration::from_millis(200),
13676            handler.handle_control_frame(
13677                &cleanup_probe_ctx,
13678                supervisor_health_probe_frame(303, "mcp"),
13679            ),
13680        )
13681        .await
13682        .expect("probe should fail immediately when the control lane is gone")
13683        .unwrap();
13684        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13685        assert_eq!(
13686            parse_error(&cleanup_response[0])["code"],
13687            "target_unavailable"
13688        );
13689        assert!(parse_error(&cleanup_response[0])["message"]
13690            .as_str()
13691            .unwrap()
13692            .contains("no module connection"));
13693
13694        handler
13695            .cleanup_connection(module_ctx.connection_id)
13696            .unwrap();
13697    }
13698
13699    #[tokio::test]
13700    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13701        let registry = Arc::new(Registry::default());
13702        let supervisor_handle = SupervisorHandle::new();
13703        let supervisor =
13704            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13705                .with_handle(supervisor_handle.clone())
13706                .with_connection_file_path(
13707                    std::env::temp_dir()
13708                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13709                );
13710        let module = supervisor
13711            .supervise_configured(
13712                ModuleSpec {
13713                    module_id: "warming".to_string(),
13714                    program: fake_aft_stub_path(),
13715                    args: Vec::new(),
13716                    env: Vec::new(),
13717                    reserved: false,
13718                    reserved_prefixes: Vec::new(),
13719                    protocol: ModuleProtocol::Subc,
13720                    overlap: Default::default(),
13721                },
13722                true,
13723            )
13724            .unwrap();
13725        assert_eq!(module.state().unwrap(), ModuleState::Running);
13726
13727        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13728        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13729        let response = handler
13730            .handle_control_frame(
13731                &ctx,
13732                route_open_frame(304, "warming", unique_project_root("warming")),
13733            )
13734            .await
13735            .unwrap();
13736        module.stop().await.unwrap();
13737
13738        assert_eq!(response[0].header.ty, FrameType::Error);
13739        let error = parse_error(&response[0]);
13740        assert_eq!(error["code"], "module_warming");
13741        assert!(error["message"]
13742            .as_str()
13743            .unwrap()
13744            .contains("state=running, enabled=true, live=false"));
13745    }
13746
13747    #[test]
13748    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13749        let handler = ControlHandler::new(Arc::new(Registry::default()));
13750        let capture = EventCapture::default();
13751        let _subscriber =
13752            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13753        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13754        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13755        let pending = (0..limit).collect::<Vec<_>>();
13756        let response = handler
13757            .route_open_capacity_refusal(
13758                &ctx,
13759                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13760                "busy",
13761                pending.len(),
13762                limit,
13763            )
13764            .unwrap();
13765        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13766        let event = capture
13767            .events()
13768            .into_iter()
13769            .find(|event| {
13770                event.target == "control"
13771                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13772            })
13773            .expect("connection admission refusal event");
13774        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13775        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13776        // Refused before `handle_route_open` parses the request, so the caller
13777        // fields are read from the frame itself.
13778        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13779        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13780    }
13781
13782    #[test]
13783    fn route_open_target_cap_logs_admission_reason_and_capacity() {
13784        let handler = ControlHandler::new(Arc::new(Registry::default()));
13785        let capture = EventCapture::default();
13786        let _subscriber =
13787            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13788        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13789        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13790        let guards = (0..limit)
13791            .map(|_| {
13792                handler
13793                    .route_bind_concurrency
13794                    .try_admit("busy", limit)
13795                    .unwrap()
13796            })
13797            .collect::<Vec<_>>();
13798        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13799            Err(in_flight) => in_flight,
13800            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13801        };
13802        let response = handler
13803            .route_open_target_capacity_refusal(
13804                &ctx,
13805                &RouteOpenCaller::unknown(),
13806                &route_open_frame(397, "busy", unique_project_root("target-cap")),
13807                "busy",
13808                in_flight,
13809            )
13810            .unwrap();
13811        assert_eq!(parse_error(&response)["code"], "target_unavailable");
13812        let event = capture
13813            .events()
13814            .into_iter()
13815            .find(|event| {
13816                event.target == "control"
13817                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13818            })
13819            .expect("target admission refusal event");
13820        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13821        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13822        drop(guards);
13823    }
13824
13825    #[cfg(unix)]
13826    #[tokio::test(start_paused = true)]
13827    async fn supervisor_list_shows_a_failed_modules_next_retry() {
13828        let registry = Arc::new(Registry::default());
13829        let handle = SupervisorHandle::new();
13830        let supervisor =
13831            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13832                .with_handle(handle.clone());
13833        let module = supervisor
13834            .spawn(crate::ModuleSpec {
13835                module_id: "retry-list".into(),
13836                program: PathBuf::from("/bin/sh"),
13837                args: vec!["-c".into(), "exit 17".into()],
13838                env: vec![],
13839                reserved: false,
13840                reserved_prefixes: vec![],
13841                protocol: ModuleProtocol::None,
13842                overlap: Default::default(),
13843            })
13844            .unwrap();
13845        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13846        while module.state().unwrap() != crate::ModuleState::Failed {
13847            assert!(
13848                std::time::Instant::now() < deadline,
13849                "waiting for budget failure"
13850            );
13851            tokio::task::yield_now().await;
13852        }
13853        let handler = ControlHandler::new(registry).with_supervisor(handle);
13854        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13855        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13856        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13857        assert_eq!(body["modules"][0]["state"], "failed");
13858        let retry = body["modules"][0]["next_retry_at_ms"]
13859            .as_u64()
13860            .expect("scheduled retry on the wire");
13861        let now = std::time::SystemTime::now()
13862            .duration_since(std::time::UNIX_EPOCH)
13863            .unwrap()
13864            .as_millis() as u64;
13865        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13866        module.set_enabled(false).await.unwrap();
13867        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13868        let replies = handler.handle_supervisor_list(frame).await.unwrap();
13869        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13870        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13871    }
13872
13873    #[tokio::test]
13874    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
13875        let _guard = crate::subtree_resources::fault_read_guard().await;
13876        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
13877        let supervisor =
13878            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13879        let module = supervisor
13880            .spawn(crate::ModuleSpec {
13881                module_id: "pid-read-budget".into(),
13882                program: std::env::current_exe().unwrap(),
13883                args: vec![
13884                    "--exact".into(),
13885                    "control::tests::pid_resource_read_fixture".into(),
13886                ],
13887                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13888                    .into_iter()
13889                    .map(|key| (key.into(), dir.display().to_string()))
13890                    .chain(std::iter::once((
13891                        "SUBC_SUBTREE_PID_FIXTURE".into(),
13892                        "1".into(),
13893                    )))
13894                    .collect(),
13895                reserved: false,
13896                reserved_prefixes: Vec::new(),
13897                protocol: ModuleProtocol::None,
13898                overlap: Default::default(),
13899            })
13900            .unwrap();
13901        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
13902        while !matches!(
13903            module.child_resource_usage(),
13904            subc_control::ChildResourceUsage::Measured(_)
13905        ) {
13906            assert!(
13907                tokio::time::Instant::now() < ready_deadline,
13908                "fixture process was not measurable"
13909            );
13910            tokio::time::sleep(Duration::from_millis(10)).await;
13911        }
13912        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
13913        let reading = crate::subtree_resources::read_process(module.clone()).await;
13914        module.stop().await.unwrap();
13915        assert!(
13916            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
13917            "{reading:?}"
13918        );
13919    }
13920
13921    #[test]
13922    fn pid_resource_read_fixture() {
13923        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
13924            loop {
13925                std::thread::park_timeout(Duration::from_secs(600));
13926            }
13927        }
13928    }
13929
13930    #[tokio::test]
13931    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
13932        let _guard = crate::subtree_resources::fault_read_guard().await;
13933        let registry = Arc::new(Registry::default());
13934        let handle = SupervisorHandle::new();
13935        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13936            .with_handle(handle.clone());
13937        let module = supervisor
13938            .supervise_configured(
13939                crate::ModuleSpec {
13940                    module_id: "memory-read".into(),
13941                    program: PathBuf::from("disabled-module"),
13942                    args: Vec::new(),
13943                    env: Vec::new(),
13944                    reserved: false,
13945                    reserved_prefixes: Vec::new(),
13946                    protocol: ModuleProtocol::Subc,
13947                    overlap: Default::default(),
13948                },
13949                false,
13950            )
13951            .unwrap();
13952        let (target, release, began) = crate::subtree_resources::held();
13953        module.set_subtree_target_for_test(target);
13954        let handler = ControlHandler::new(registry).with_supervisor(handle);
13955        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
13956        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13957        let replies = handler
13958            .handle_client_control_request(
13959                &ctx,
13960                frame,
13961                ClientControlRequest::SupervisorList {},
13962                &mut None,
13963            )
13964            .await
13965            .unwrap();
13966        assert!(
13967            replies.is_empty(),
13968            "list reply must be detached from the connection reader"
13969        );
13970        began.await.unwrap();
13971        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13972        let replies = handler
13973            .handle_client_control_request(
13974                &ctx,
13975                frame,
13976                ClientControlRequest::SupervisorSpawnSnapshot {},
13977                &mut None,
13978            )
13979            .await
13980            .unwrap();
13981        assert_eq!(
13982            replies.len(),
13983            1,
13984            "another request must complete while the subtree read stalls"
13985        );
13986        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
13987            .await
13988            .unwrap()
13989            .unwrap();
13990        release.send(()).unwrap();
13991        drop(release);
13992        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
13993        assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
13994        assert_eq!(body["modules"][0]["subtree"]["reason"], "timeout");
13995        assert_eq!(
13996            body["modules"][0]["subtree"]["process_count"], 1,
13997            "a count obtained before the stalled read must survive timeout"
13998        );
13999        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14000        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14001        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14002        assert_eq!(
14003            body["modules"][0]["subtree"]["source"], "process_sum",
14004            "the next request must read afresh"
14005        );
14006    }
14007
14008    #[tokio::test]
14009    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14010        let registry = Arc::new(Registry::default());
14011        let handle = SupervisorHandle::new();
14012        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14013            .with_handle(handle.clone());
14014        supervisor
14015            .supervise_configured(
14016                crate::ModuleSpec {
14017                    module_id: "drift".into(),
14018                    program: PathBuf::from("disabled-module"),
14019                    args: Vec::new(),
14020                    env: Vec::new(),
14021                    reserved: false,
14022                    reserved_prefixes: Vec::new(),
14023                    protocol: ModuleProtocol::Subc,
14024                    overlap: Default::default(),
14025                },
14026                false,
14027            )
14028            .unwrap();
14029        let handler = ControlHandler::new(registry).with_supervisor(handle);
14030        for configured in [None, Some(false), Some(true)] {
14031            handler.capability_evaluator.configure(
14032                configured.map(|enabled| ("drift".to_string(), enabled)),
14033                BTreeMap::new(),
14034            );
14035            let frame =
14036                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14037            let replies = handler
14038                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14039                .await
14040                .unwrap();
14041            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14042            assert!(body.get("configured_enabled").is_none());
14043            assert_eq!(body["op"], "supervisor.ack");
14044            let frame =
14045                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14046            let replies = handler.handle_supervisor_list(frame).await.unwrap();
14047            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14048            assert_eq!(
14049                body["modules"][0].get("configured_enabled"),
14050                configured.map(Value::Bool).as_ref()
14051            );
14052            assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14053            assert_eq!(body["modules"][0]["subtree"]["reason"], "not_running");
14054        }
14055    }
14056
14057    #[tokio::test(flavor = "current_thread")]
14058    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14059        let handler = ControlHandler::new(Arc::new(Registry::default()));
14060        let capture = EventCapture::default();
14061        let _subscriber =
14062            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14063        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14064        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14065
14066        let _ = handler
14067            .handle_client_control_request(
14068                &ctx,
14069                frame,
14070                ClientControlRequest::SupervisorSetEnabled {
14071                    module_id: "broca".to_string(),
14072                    enabled: false,
14073                },
14074                &mut None,
14075            )
14076            .await
14077            .unwrap();
14078
14079        let events = capture
14080            .events()
14081            .into_iter()
14082            .filter(|event| {
14083                event.target == "control"
14084                    && event.fields.get("message").map(String::as_str)
14085                        == Some("supervisor request received")
14086            })
14087            .collect::<Vec<_>>();
14088        assert_eq!(events.len(), 1, "one supervisor request log line");
14089        let event = &events[0];
14090        assert_eq!(event.level, tracing::Level::INFO);
14091        assert_eq!(
14092            event.fields.get("op"),
14093            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14094        );
14095        assert_eq!(
14096            event.fields.get("module_id"),
14097            Some(&"\"broca\"".to_string())
14098        );
14099        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14100        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14101        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14102    }
14103
14104    #[tokio::test(flavor = "current_thread")]
14105    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14106        let registry = Arc::new(Registry::default());
14107        let module_connection = ConnectionId::new(303);
14108        registry
14109            .register_with_control_ops(
14110                manifest("aft", PROTOCOL_VERSION),
14111                PROTOCOL_VERSION,
14112                module_connection,
14113                module_baseline_control_ops(),
14114            )
14115            .unwrap();
14116        let handler = ControlHandler::new(registry);
14117        let capture = EventCapture::default();
14118        let _subscriber =
14119            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14120        let (ctx, _rx) = route_ctx(module_connection);
14121        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14122
14123        let _ = handler
14124            .handle_client_control_request(
14125                &ctx,
14126                frame,
14127                ClientControlRequest::SupervisorSetEnabled {
14128                    module_id: "broca".to_string(),
14129                    enabled: false,
14130                },
14131                &mut None,
14132            )
14133            .await
14134            .unwrap();
14135
14136        let callers = capture
14137            .events()
14138            .into_iter()
14139            .filter(|event| {
14140                event.fields.get("message").map(String::as_str)
14141                    == Some("supervisor request received")
14142            })
14143            .map(|event| event.fields.get("caller").cloned())
14144            .collect::<Vec<_>>();
14145        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14146    }
14147
14148    #[tokio::test(flavor = "current_thread")]
14149    async fn supervisor_list_does_not_log_request_received() {
14150        let handler = ControlHandler::new(Arc::new(Registry::default()));
14151        let capture = EventCapture::default();
14152        let _subscriber =
14153            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14154        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14155        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14156
14157        handler
14158            .handle_client_control_request(
14159                &ctx,
14160                frame,
14161                ClientControlRequest::SupervisorList {},
14162                &mut None,
14163            )
14164            .await
14165            .unwrap();
14166
14167        assert!(capture.events().into_iter().all(|event| {
14168            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14169        }));
14170    }
14171
14172    #[tokio::test(flavor = "current_thread")]
14173    async fn supervisor_rescan_preview_does_not_log_request_received() {
14174        let handler = ControlHandler::new(Arc::new(Registry::default()));
14175        let capture = EventCapture::default();
14176        let _subscriber =
14177            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14178        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14179        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14180
14181        handler
14182            .handle_client_control_request(
14183                &ctx,
14184                frame,
14185                ClientControlRequest::SupervisorRescan { preview: true },
14186                &mut None,
14187            )
14188            .await
14189            .unwrap();
14190
14191        assert!(capture.events().into_iter().all(|event| {
14192            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14193        }));
14194    }
14195
14196    struct ApproveOperator;
14197    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14198        fn prompt(
14199            &self,
14200            _: &str,
14201            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14202        ) -> crate::operator_confirm::ProviderResult {
14203            crate::operator_confirm::ProviderResult::Approved
14204        }
14205    }
14206
14207    #[tokio::test]
14208    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14209        assert_eq!(
14210            ControlHandler::default().privileged_operator_prompt,
14211            PrivilegedOperatorPrompt::Log
14212        );
14213        for (initial, changed, running) in [
14214            ("log", "enforce", PrivilegedOperatorPrompt::Log),
14215            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14216        ] {
14217            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14218            let path = dir.join("subc.jsonc");
14219            std::fs::write(
14220                &path,
14221                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14222            )
14223            .unwrap();
14224            let config = crate::daemon_config::load(&path).unwrap().unwrap();
14225            let registry = Arc::new(Registry::default());
14226            let handle = SupervisorHandle::new();
14227            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14228                .with_handle(handle.clone());
14229            let handler = ControlHandler::new(registry)
14230                .with_supervisor(handle)
14231                .with_privileged_operator_prompt(config.privileged_operator_prompt)
14232                .with_supervisor_rescan(supervisor, &path, config.port);
14233            handler.forwarding.operator_confirms().configure(
14234                Arc::new(ApproveOperator),
14235                crate::operator_confirm::OperatorLimits::default(),
14236            );
14237            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14238
14239            for (disk_mode, needs_restart) in [
14240                (Some(initial), false),
14241                (Some(changed), true),
14242                (Some(changed), true),
14243                (None, running == PrivilegedOperatorPrompt::Enforce),
14244                (Some(initial), false),
14245            ] {
14246                let mut doc = json!({"version":1});
14247                if let Some(mode) = disk_mode {
14248                    doc["privileged_operator_prompt"] = json!(mode);
14249                }
14250                std::fs::write(&path, doc.to_string()).unwrap();
14251                for preview in [true, false] {
14252                    let request = ClientControlRequest::SupervisorRescan { preview };
14253                    let frame = Frame::build(
14254                        FrameType::Request,
14255                        control_flags(),
14256                        0,
14257                        0,
14258                        1,
14259                        serde_json::to_vec(&request).unwrap(),
14260                    )
14261                    .unwrap();
14262                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14263                    assert_eq!(frames.len(), 1);
14264                    let ClientControlResponse::SupervisorRescan { result } =
14265                        serde_json::from_slice(&frames[0].body).unwrap()
14266                    else {
14267                        panic!("expected a rescan response");
14268                    };
14269                    let expected = if needs_restart {
14270                        vec!["privileged_operator_prompt"]
14271                    } else {
14272                        Vec::new()
14273                    };
14274                    assert_eq!(result.restart_required, expected);
14275                    assert_eq!(handler.privileged_operator_prompt, running);
14276                    assert_eq!(
14277                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14278                        running
14279                    );
14280                }
14281            }
14282        }
14283    }
14284
14285    #[tokio::test]
14286    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14287        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14288        let path = dir.join("subc.jsonc");
14289        let registry = Arc::new(Registry::default());
14290        let handle = SupervisorHandle::new();
14291        let supervisor =
14292            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14293        let handler = ControlHandler::new(registry)
14294            .with_supervisor(handle)
14295            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14296            .with_supervisor_rescan(supervisor, &path, None);
14297        handler.forwarding.operator_confirms().configure(
14298            Arc::new(ApproveOperator),
14299            crate::operator_confirm::OperatorLimits::default(),
14300        );
14301        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14302        for preview in [true, false] {
14303            std::fs::write(
14304                &path,
14305                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14306            )
14307            .unwrap();
14308            let request = ClientControlRequest::SupervisorRescan { preview };
14309            let frame = Frame::build(
14310                FrameType::Request,
14311                control_flags(),
14312                0,
14313                0,
14314                1,
14315                serde_json::to_vec(&request).unwrap(),
14316            )
14317            .unwrap();
14318            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14319            assert_eq!(frames.len(), 1);
14320            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14321            assert_eq!(
14322                handler.privileged_operator_prompt,
14323                PrivilegedOperatorPrompt::Enforce
14324            );
14325        }
14326    }
14327
14328    /// One wire code has several senders, so the refusal line names the check
14329    /// that refused. This drives the shared refusal path for ordinary refusals
14330    /// with an unregistered
14331    /// target and requires the branch label on the event.
14332    #[tokio::test]
14333    async fn route_open_refusal_names_the_check_that_refused() {
14334        let handler = ControlHandler::new(Arc::new(Registry::default()));
14335        let capture = EventCapture::default();
14336        let _subscriber =
14337            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14338        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14339        let response = handler
14340            .handle_control_frame(
14341                &ctx,
14342                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14343            )
14344            .await
14345            .unwrap();
14346
14347        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14348        let event = capture
14349            .events()
14350            .into_iter()
14351            .find(|event| {
14352                event.target == "control"
14353                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14354            })
14355            .expect("route.open refusal event");
14356        assert_eq!(
14357            event.fields.get("reason"),
14358            Some(&"\"not_registered\"".to_string())
14359        );
14360    }
14361
14362    #[tokio::test]
14363    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14364        let registry = Arc::new(Registry::default());
14365        let supervisor_handle = SupervisorHandle::new();
14366        let supervisor =
14367            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14368                .with_handle(supervisor_handle.clone())
14369                .with_connection_file_path(std::env::temp_dir().join(format!(
14370                    "subc-route-open-refusal-info-{}",
14371                    std::process::id()
14372                )));
14373        let module = supervisor
14374            .supervise_configured(
14375                ModuleSpec {
14376                    module_id: "warming".to_string(),
14377                    program: fake_aft_stub_path(),
14378                    args: Vec::new(),
14379                    env: Vec::new(),
14380                    reserved: false,
14381                    reserved_prefixes: Vec::new(),
14382                    protocol: ModuleProtocol::Subc,
14383                    overlap: Default::default(),
14384                },
14385                true,
14386            )
14387            .unwrap();
14388        assert_eq!(module.state().unwrap(), ModuleState::Running);
14389
14390        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14391        assert!(handler
14392            .counters()
14393            .snapshot()
14394            .get("route_open_refused_by_code")
14395            .is_none());
14396        let capture = EventCapture::default();
14397        let _subscriber =
14398            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14399        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14400        let response = handler
14401            .handle_control_frame(
14402                &ctx,
14403                route_open_frame(394, "warming", unique_project_root("refusal-info")),
14404            )
14405            .await
14406            .unwrap();
14407        module.stop().await.unwrap();
14408
14409        assert_eq!(parse_error(&response[0])["code"], "module_warming");
14410        let event = capture
14411            .events()
14412            .into_iter()
14413            .find(|event| {
14414                event.target == "control"
14415                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14416            })
14417            .expect("route.open refusal event");
14418        assert_eq!(
14419            event.fields.get("module_id"),
14420            Some(&"\"warming\"".to_string())
14421        );
14422        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14423        assert_eq!(
14424            event.fields.get("reason"),
14425            Some(&"\"supervised_not_registered\"".to_string())
14426        );
14427        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14428        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14429        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14430        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14431        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14432        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14433        assert_eq!(
14434            handler.counters().snapshot()["route_open_refused_by_code"],
14435            json!({ "module_warming": 1 })
14436        );
14437    }
14438
14439    const OUTAGE_START: &str = "route.open refusing module: not serving";
14440    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14441
14442    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14443        capture
14444            .events()
14445            .into_iter()
14446            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14447            .collect()
14448    }
14449
14450    fn supervise_stub(
14451        registry: &Arc<Registry>,
14452        module_id: &str,
14453        enabled: bool,
14454    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14455        let supervisor_handle = SupervisorHandle::new();
14456        let supervisor =
14457            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14458                .with_handle(supervisor_handle.clone())
14459                .with_connection_file_path(std::env::temp_dir().join(format!(
14460                    "subc-route-outage-{module_id}-{}",
14461                    std::process::id()
14462                )));
14463        let module = supervisor
14464            .supervise_configured(
14465                ModuleSpec {
14466                    module_id: module_id.to_string(),
14467                    program: fake_aft_stub_path(),
14468                    args: Vec::new(),
14469                    env: Vec::new(),
14470                    reserved: false,
14471                    reserved_prefixes: Vec::new(),
14472                    protocol: ModuleProtocol::Subc,
14473                    overlap: Default::default(),
14474                },
14475                enabled,
14476            )
14477            .unwrap();
14478        (supervisor_handle, module)
14479    }
14480
14481    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14482        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14483            module_id: module_id.to_string(),
14484            drain_timeout_ms: Some(50),
14485        })
14486        .unwrap();
14487        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14488    }
14489
14490    /// Two handlers built over one forwarding table must share one outage
14491    /// tracker; separate trackers would each log their own opening line for
14492    /// the same outage.
14493    #[test]
14494    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14495        let registry = Arc::new(Registry::default());
14496        let forwarding = Arc::new(ForwardingTable::default());
14497        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14498        let second = ControlHandler::with_forwarding(registry, forwarding);
14499        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14500    }
14501
14502    /// A client can name any module id it likes. Refusing an unknown one,
14503    /// however often, must not create outage state or outage lines, or the
14504    /// tracker would be a memory sink any client could fill.
14505    #[tokio::test(flavor = "current_thread")]
14506    async fn route_open_unknown_module_refusals_add_no_outage_state() {
14507        let handler = ControlHandler::new(Arc::new(Registry::default()));
14508        let capture = EventCapture::default();
14509        let _subscriber =
14510            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14511        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14512        for corr in 0..8 {
14513            let response = handler
14514                .handle_control_frame(
14515                    &ctx,
14516                    route_open_frame(
14517                        380 + corr,
14518                        &format!("nobody-{corr}"),
14519                        unique_project_root("outage-unknown"),
14520                    ),
14521                )
14522                .await
14523                .unwrap();
14524            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14525        }
14526
14527        assert_eq!(handler.route_outages.tracked_module_count(), 0);
14528        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14529        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14530    }
14531
14532    /// Drives the refusal path end to end: a supervised module that served
14533    /// before and stopped being registered with no instruction to stop is a
14534    /// WARN, and the same module refused after an operator `supervisor.restart`
14535    /// is an INFO.
14536    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14537    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14538        let registry = Arc::new(Registry::default());
14539        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
14540        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14541        let capture = EventCapture::default();
14542        let _subscriber =
14543            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14544        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14545        // The stub never registers, so pretend it served once: otherwise every
14546        // refusal would fall in its startup window.
14547        handler.route_outages.record_accepted("outage-restart");
14548
14549        let response = handler
14550            .handle_control_frame(
14551                &ctx,
14552                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14553            )
14554            .await
14555            .unwrap();
14556        assert_eq!(response[0].header.ty, FrameType::Error);
14557        let starts = outage_lines(&capture, OUTAGE_START);
14558        assert_eq!(starts.len(), 1, "{starts:?}");
14559        assert_eq!(starts[0].level, tracing::Level::WARN);
14560        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14561        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14562        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14563        handler.route_outages.record_accepted("outage-restart");
14564        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14565
14566        let restart = handler
14567            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14568            .await
14569            .unwrap();
14570        assert_eq!(
14571            restart[0].header.ty,
14572            FrameType::Response,
14573            "{:?}",
14574            parse_error(&restart[0])
14575        );
14576        handler
14577            .handle_control_frame(
14578                &ctx,
14579                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14580            )
14581            .await
14582            .unwrap();
14583        module.stop().await.unwrap();
14584
14585        let starts = outage_lines(&capture, OUTAGE_START);
14586        assert_eq!(starts.len(), 2, "{starts:?}");
14587        assert_eq!(starts[1].level, tracing::Level::INFO);
14588        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14589    }
14590
14591    /// A restart refused before it touched the module (here: the module is
14592    /// disabled) must clear its operator mark, so the next real outage is
14593    /// still reported as a warning.
14594    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14595    async fn failed_operator_restart_leaves_no_operator_mark() {
14596        let registry = Arc::new(Registry::default());
14597        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
14598        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14599        let capture = EventCapture::default();
14600        let _subscriber =
14601            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14602        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14603        handler.route_outages.record_accepted("outage-disabled");
14604
14605        let restart = handler
14606            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14607            .await
14608            .unwrap();
14609        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14610        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14611
14612        handler
14613            .handle_control_frame(
14614                &ctx,
14615                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14616            )
14617            .await
14618            .unwrap();
14619        let starts = outage_lines(&capture, OUTAGE_START);
14620        assert_eq!(starts.len(), 1, "{starts:?}");
14621        assert_eq!(starts[0].level, tracing::Level::WARN);
14622    }
14623
14624    #[tokio::test(flavor = "current_thread")]
14625    async fn route_open_unknown_module_escapes_target_module_id() {
14626        let handler = ControlHandler::new(Arc::new(Registry::default()));
14627        let capture = EventCapture::default();
14628        let _subscriber =
14629            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14630        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14631        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14632        let response = handler
14633            .handle_control_frame(
14634                &ctx,
14635                route_open_frame(
14636                    395,
14637                    hostile_module_id,
14638                    unique_project_root("hostile-target-module-id"),
14639                ),
14640            )
14641            .await
14642            .unwrap();
14643
14644        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14645        let event = capture
14646            .events()
14647            .into_iter()
14648            .find(|event| {
14649                event.target == "control"
14650                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14651            })
14652            .expect("route.open unknown-module refusal event");
14653        let logged = event.fields.get("module_id").expect("module_id field");
14654        assert!(!logged.bytes().any(|byte| byte < 0x20));
14655        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14656    }
14657
14658    /// A route.open refusal is the daemon's own answer, so its ERROR carries
14659    /// the daemon-origin bit on the frame the client receives.
14660    #[tokio::test(flavor = "current_thread")]
14661    async fn route_open_refusal_carries_daemon_origin() {
14662        let handler = ControlHandler::new(Arc::new(Registry::default()));
14663        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14664        let response = handler
14665            .handle_control_frame(
14666                &ctx,
14667                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14668            )
14669            .await
14670            .unwrap();
14671
14672        assert_eq!(response.len(), 1);
14673        assert_eq!(response[0].header.ty, FrameType::Error);
14674        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14675        assert!(
14676            response[0].header.flags.is_daemon_origin(),
14677            "route.open refusal must carry daemon origin: {:?}",
14678            response[0].header.flags
14679        );
14680    }
14681
14682    /// The drain `module_reloading` refusal is daemon-synthesized for an
14683    /// endpoint that is reloading, so it carries the daemon-origin bit on the
14684    /// frame the client receives.
14685    #[tokio::test(flavor = "current_thread")]
14686    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14687        let registry = Arc::new(Registry::default());
14688        let forwarding = Arc::new(ForwardingTable::default());
14689        let handler =
14690            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14691        let module_connection = ConnectionId::new(95);
14692        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14693        hello_via_sink(
14694            &handler,
14695            &module_ctx,
14696            &mut module_rx,
14697            hello_frame("aft", PROTOCOL_VERSION, 395),
14698        )
14699        .await;
14700        forwarding
14701            .begin_module_drain("aft", RouteCloseReason::Restart)
14702            .unwrap()
14703            .expect("the module endpoint drains");
14704
14705        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14706        let response = handler
14707            .handle_control_frame(
14708                &client_ctx,
14709                route_open_frame(396, "aft", unique_project_root("drain-origin")),
14710            )
14711            .await
14712            .unwrap();
14713
14714        assert_eq!(response.len(), 1);
14715        assert_eq!(response[0].header.ty, FrameType::Error);
14716        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14717        assert!(
14718            response[0].header.flags.is_daemon_origin(),
14719            "drain module_reloading must carry daemon origin: {:?}",
14720            response[0].header.flags
14721        );
14722    }
14723
14724    /// A module that refuses a bind with a daemon code still cannot mint a
14725    /// daemon refusal: the relayed `module_reloading` body reaches the client
14726    /// with the daemon-origin bit clear.
14727    #[tokio::test(flavor = "current_thread")]
14728    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14729        let registry = Arc::new(Registry::default());
14730        let forwarding = Arc::new(ForwardingTable::default());
14731        let handler =
14732            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14733        let module_connection = ConnectionId::new(95);
14734        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14735        hello_via_sink(
14736            &handler,
14737            &module_ctx,
14738            &mut module_rx,
14739            hello_frame("aft", PROTOCOL_VERSION, 395),
14740        )
14741        .await;
14742
14743        let client_connection = ConnectionId::new(96);
14744        let (client_ctx, _client_rx) = route_ctx(client_connection);
14745        let (route_task, bind) = relay_route_open(
14746            &handler,
14747            client_connection,
14748            &client_ctx.egress,
14749            &mut module_rx,
14750            396,
14751            "aft",
14752            "spoofed-reloading",
14753        )
14754        .await;
14755        // The module sets the origin bit on its own rejection too: neither
14756        // the bit nor the daemon code may survive the relay as daemon-owned.
14757        let rejection = Frame::build(
14758            FrameType::Error,
14759            crate::router::daemon_error_flags(),
14760            0,
14761            0,
14762            bind.header.corr,
14763            serde_json::to_vec(&ErrorBody::new(
14764                "module_reloading",
14765                "module refused route.bind",
14766            ))
14767            .unwrap(),
14768        )
14769        .unwrap();
14770        handler
14771            .handle_control_frame(&module_ctx, rejection)
14772            .await
14773            .unwrap();
14774
14775        let response = route_task.await.unwrap();
14776        assert_eq!(response.len(), 1);
14777        assert_eq!(response[0].header.ty, FrameType::Error);
14778        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14779        assert!(
14780            !response[0].header.flags.is_daemon_origin(),
14781            "a module's own rejection must not carry daemon origin: {:?}",
14782            response[0].header.flags
14783        );
14784    }
14785
14786    #[tokio::test(flavor = "current_thread")]
14787    async fn route_open_module_rejection_uses_daemon_counter_key() {
14788        let registry = Arc::new(Registry::default());
14789        let forwarding = Arc::new(ForwardingTable::default());
14790        let handler =
14791            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14792        let module_connection = ConnectionId::new(95);
14793        let (module_ctx, mut module_rx) = route_ctx(module_connection);
14794        hello_via_sink(
14795            &handler,
14796            &module_ctx,
14797            &mut module_rx,
14798            hello_frame("aft", PROTOCOL_VERSION, 395),
14799        )
14800        .await;
14801
14802        let client_connection = ConnectionId::new(96);
14803        let (client_ctx, _client_rx) = route_ctx(client_connection);
14804        let capture = EventCapture::default();
14805        let _subscriber =
14806            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14807        let (route_task, bind) = relay_route_open(
14808            &handler,
14809            client_connection,
14810            &client_ctx.egress,
14811            &mut module_rx,
14812            396,
14813            "aft",
14814            "hostile-module-code",
14815        )
14816        .await;
14817        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14818        let rejection = Frame::build(
14819            FrameType::Error,
14820            control_flags(),
14821            0,
14822            0,
14823            bind.header.corr,
14824            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14825        )
14826        .unwrap();
14827        handler
14828            .handle_control_frame(&module_ctx, rejection)
14829            .await
14830            .unwrap();
14831
14832        let response = route_task.await.unwrap();
14833        assert_eq!(parse_error(&response[0])["code"], hostile_code);
14834        let counters = handler.counters().snapshot();
14835        assert_eq!(
14836            counters["route_open_refused_by_code"],
14837            json!({ "module_rejected": 1 })
14838        );
14839        assert!(counters["route_open_refused_by_code"]
14840            .get(hostile_code)
14841            .is_none());
14842
14843        let event = capture
14844            .events()
14845            .into_iter()
14846            .find(|event| {
14847                event.target == "control"
14848                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14849            })
14850            .expect("route.open module-rejection refusal event");
14851        let logged = event.fields.get("module_code").expect("module_code field");
14852        assert!(!logged.bytes().any(|byte| byte < 0x20));
14853        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14854    }
14855
14856    #[tokio::test]
14857    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14858        let registry = Arc::new(Registry::default());
14859        let supervisor_handle = SupervisorHandle::new();
14860        let missing_program = std::env::temp_dir().join(format!(
14861            "subc-route-open-missing-program-{}",
14862            std::process::id()
14863        ));
14864        let supervisor =
14865            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14866                .with_handle(supervisor_handle.clone());
14867        let module = supervisor
14868            .supervise_configured(
14869                ModuleSpec {
14870                    module_id: "failed".to_string(),
14871                    program: missing_program,
14872                    args: Vec::new(),
14873                    env: Vec::new(),
14874                    reserved: false,
14875                    reserved_prefixes: Vec::new(),
14876                    protocol: ModuleProtocol::Subc,
14877                    overlap: Default::default(),
14878                },
14879                true,
14880            )
14881            .unwrap();
14882        assert_eq!(module.state().unwrap(), ModuleState::Failed);
14883
14884        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14885        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14886        let response = handler
14887            .handle_control_frame(
14888                &ctx,
14889                route_open_frame(305, "failed", unique_project_root("failed")),
14890            )
14891            .await
14892            .unwrap();
14893
14894        assert_eq!(response[0].header.ty, FrameType::Error);
14895        let error = parse_error(&response[0]);
14896        assert_eq!(error["code"], "target_unavailable");
14897        assert!(error["message"]
14898            .as_str()
14899            .unwrap()
14900            .contains("state=failed, enabled=true, live=false"));
14901    }
14902
14903    #[tokio::test]
14904    async fn route_open_role_mismatch_remains_target_unavailable() {
14905        let registry = Arc::new(Registry::default());
14906        let handler = ControlHandler::new(Arc::clone(&registry));
14907        handler
14908            .handle_control(
14909                ConnectionId::new(41),
14910                non_routable_hello_frame_with_control_ops("health-only", 306, None),
14911            )
14912            .unwrap();
14913
14914        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14915        let response = handler
14916            .handle_control_frame(
14917                &ctx,
14918                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14919            )
14920            .await
14921            .unwrap();
14922
14923        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14924        assert!(parse_error(&response[0])["message"]
14925            .as_str()
14926            .unwrap()
14927            .contains("does not provide the requested target"));
14928    }
14929
14930    #[tokio::test]
14931    async fn route_open_inactive_registration_remains_target_unavailable() {
14932        let registry = Arc::new(Registry::default());
14933        let handler = ControlHandler::new(Arc::clone(&registry));
14934        handler
14935            .handle_control(
14936                ConnectionId::new(43),
14937                hello_frame("inactive", PROTOCOL_VERSION, 308),
14938            )
14939            .unwrap();
14940        assert!(registry
14941            .set_module_state_for_test("inactive", ChannelState::Closed)
14942            .unwrap());
14943
14944        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14945        let response = handler
14946            .handle_control_frame(
14947                &ctx,
14948                route_open_frame(309, "inactive", unique_project_root("inactive")),
14949            )
14950            .await
14951            .unwrap();
14952
14953        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14954        assert!(parse_error(&response[0])["message"]
14955            .as_str()
14956            .unwrap()
14957            .contains("is not active"));
14958    }
14959
14960    #[tokio::test]
14961    async fn late_health_reply_is_recorded_through_the_module_response_path() {
14962        let registry = Arc::new(Registry::default());
14963        let forwarding = Arc::new(ForwardingTable::default());
14964        let supervisor_handle = SupervisorHandle::new();
14965        let supervisor =
14966            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
14967                .with_forwarding(Arc::clone(&forwarding))
14968                .with_handle(supervisor_handle.clone());
14969        let module = supervisor
14970            .supervise_configured(
14971                crate::ModuleSpec {
14972                    module_id: "late-health-response".to_string(),
14973                    program: PathBuf::from("disabled-module"),
14974                    args: Vec::new(),
14975                    env: Vec::new(),
14976                    reserved: false,
14977                    reserved_prefixes: Vec::new(),
14978                    protocol: ModuleProtocol::Subc,
14979                    overlap: Default::default(),
14980                },
14981                false,
14982            )
14983            .unwrap();
14984        let handler =
14985            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14986                .with_supervisor(supervisor_handle);
14987        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14988        handler
14989            .handle_control_frame(
14990                &module_ctx,
14991                hello_frame_with_control_ops(
14992                    "late-health-response",
14993                    PROTOCOL_VERSION,
14994                    7,
14995                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14996                ),
14997            )
14998            .await
14999            .unwrap();
15000        let probe_started_at = Instant::now() - Duration::from_millis(80);
15001        let pending = forwarding
15002            .begin_health_probe_rpc_for(
15003                "late-health-response",
15004                MODULE_CONTROL_OP_HEALTH_CHECK,
15005                probe_started_at,
15006                Instant::now() - Duration::from_millis(1),
15007            )
15008            .unwrap();
15009        assert!(forwarding
15010            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15011            .unwrap());
15012
15013        let responses = handler
15014            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15015            .await
15016            .unwrap();
15017
15018        assert!(responses.is_empty());
15019        let health = module.status().unwrap().health;
15020        assert_eq!(health.late_answer_count, 1);
15021        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15022    }
15023
15024    #[tokio::test]
15025    async fn health_probe_timeout_and_module_death_are_typed() {
15026        let registry = Arc::new(Registry::default());
15027        let forwarding = Arc::new(ForwardingTable::default());
15028        let handler =
15029            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15030                .with_health_probe_timeout(Duration::from_millis(50));
15031        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15032        hello_via_sink(
15033            &handler,
15034            &module_ctx,
15035            &mut module_rx,
15036            hello_frame_with_control_ops(
15037                "aft",
15038                PROTOCOL_VERSION,
15039                7,
15040                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15041            ),
15042        )
15043        .await;
15044
15045        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15046        let responses = handler
15047            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15048            .await
15049            .unwrap();
15050        assert_eq!(responses[0].header.ty, FrameType::Error);
15051        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15052        let _ = module_rx.try_recv();
15053
15054        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15055        let health_handler = handler.clone();
15056        let death_task = tokio::spawn(async move {
15057            health_handler
15058                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15059                .await
15060                .unwrap()
15061        });
15062        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15063            .await
15064            .unwrap()
15065            .unwrap();
15066        handler
15067            .cleanup_connection(module_ctx.connection_id)
15068            .unwrap();
15069        let responses = death_task.await.unwrap();
15070        assert_eq!(responses[0].header.ty, FrameType::Error);
15071        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15072    }
15073
15074    #[test]
15075    fn hello_requires_exact_protocol_version() {
15076        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15077            let registry = Arc::new(Registry::default());
15078            let handler = ControlHandler::new(Arc::clone(&registry));
15079            let responses = handler
15080                .handle_control(
15081                    ConnectionId::new(connection),
15082                    hello_frame("aft", offered, 9),
15083                )
15084                .unwrap();
15085
15086            assert_eq!(responses.len(), 1);
15087            assert_eq!(responses[0].header.ty, FrameType::Error);
15088            let error = parse_error(&responses[0]);
15089            assert_eq!(error["code"], "version_unsupported");
15090            assert!(registry.get_module("aft").unwrap().is_none());
15091            assert_eq!(registry.active_registration_count().unwrap(), 0);
15092        }
15093    }
15094
15095    #[test]
15096    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15097        let registry = Arc::new(Registry::default());
15098        let forwarding = Arc::new(ForwardingTable::default());
15099        let handler =
15100            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15101        let module_connection = ConnectionId::new(301);
15102        let registration = registry
15103            .register_with_control_ops(
15104                manifest("aft-push", PROTOCOL_VERSION),
15105                PROTOCOL_VERSION,
15106                module_connection,
15107                module_baseline_control_ops(),
15108            )
15109            .unwrap();
15110        let (module_tx, _module_rx) = mpsc::channel(8);
15111        let endpoint = forwarding
15112            .register_module_connection(
15113                module_connection,
15114                "aft-push".to_string(),
15115                PROTOCOL_VERSION,
15116                manifest_concurrency(&registration.manifest),
15117                FrameSink::new(module_tx),
15118            )
15119            .unwrap();
15120
15121        // A push op this version does not know is ignored (forward-compat), not errored.
15122        let unknown = Frame::build(
15123            FrameType::Push,
15124            control_flags(),
15125            0,
15126            0,
15127            5,
15128            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15129        )
15130        .unwrap();
15131        let out = handler.handle_status_update(endpoint, unknown).unwrap();
15132        assert!(
15133            out.is_empty(),
15134            "unknown push op must be ignored, got {out:?}"
15135        );
15136
15137        // A malformed body for a KNOWN op is a real error worth surfacing.
15138        let malformed = Frame::build(
15139            FrameType::Push,
15140            control_flags(),
15141            0,
15142            0,
15143            6,
15144            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15145        )
15146        .unwrap();
15147        let out = handler.handle_status_update(endpoint, malformed).unwrap();
15148        assert_eq!(out.len(), 1);
15149        assert_eq!(out[0].header.ty, FrameType::Error);
15150        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15151    }
15152
15153    #[test]
15154    fn hello_rejected_when_connection_already_owns_client_routes() {
15155        let registry = Arc::new(Registry::default());
15156        let forwarding = Arc::new(ForwardingTable::default());
15157        let handler =
15158            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15159        // Commits a client route on connection 202 (bound to a module on conn 101).
15160        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
15161        let client_connection = ConnectionId::new(202);
15162
15163        // That same connection now tries to register as a module: rejected, so one
15164        // connection never holds both client-route and module-endpoint state.
15165        let responses = handler
15166            .handle_control(
15167                client_connection,
15168                hello_frame("aft-second", PROTOCOL_VERSION, 9),
15169            )
15170            .unwrap();
15171        assert_eq!(responses[0].header.ty, FrameType::Error);
15172        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15173        assert!(registry.get_module("aft-second").unwrap().is_none());
15174    }
15175
15176    #[tokio::test]
15177    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15178        let registry = Arc::new(Registry::default());
15179        let forwarding = Arc::new(ForwardingTable::default());
15180        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15181        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15182        hello_via_sink(
15183            &handler,
15184            &module_ctx,
15185            &mut module_rx,
15186            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15187        )
15188        .await;
15189        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15190        let pending = forwarding
15191            .begin_route_bind_relay_for_test(
15192                client_ctx.connection_id,
15193                client_ctx.egress.clone(),
15194                2,
15195                "alpha",
15196            )
15197            .unwrap();
15198        forwarding
15199            .complete_pending_relay(
15200                module_ctx.connection_id,
15201                pending.corr,
15202                RouteBindRelayOutcome::Accepted,
15203            )
15204            .unwrap();
15205        client_rx.try_recv().unwrap();
15206        for module_id in ["beta", "alpha"] {
15207            let replies = handler
15208                .handle_control_frame(
15209                    &module_ctx,
15210                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15211                )
15212                .await
15213                .unwrap();
15214            assert_eq!(replies.len(), 1, "second HELLO must be refused");
15215            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15216        }
15217        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15218        assert!(registry.get_module("beta").unwrap().is_none());
15219        assert!(matches!(
15220            forwarding
15221                .lookup_data_route(
15222                    client_ctx.connection_id,
15223                    pending.client_channel,
15224                    pending.client_epoch,
15225                )
15226                .unwrap(),
15227            DataRoute::Client(DataRouteState::Bound(_))
15228        ));
15229        assert!(handler
15230            .hello_launch_nonces
15231            .lock()
15232            .unwrap()
15233            .presented(module_ctx.connection_id, Some("alpha-nonce")));
15234        assert!(module_rx.try_recv().is_err());
15235    }
15236
15237    #[test]
15238    fn reserved_module_hello_requires_matching_launch_nonce() {
15239        let registry = Arc::new(Registry::default());
15240        let supervisor = SupervisorHandle::new();
15241        // The supervisor recorded the nonce it injected when it spawned the reserved
15242        // module; the HELLO verifier checks against the same shared handle.
15243        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15244        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15245
15246        // A HELLO with NO nonce is rejected.
15247        let no_nonce = handler
15248            .handle_control(
15249                ConnectionId::new(1),
15250                hello_frame("vault", PROTOCOL_VERSION, 1),
15251            )
15252            .unwrap();
15253        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15254        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15255        assert!(registry.get_module("vault").unwrap().is_none());
15256
15257        // A HELLO with the WRONG nonce is rejected.
15258        let wrong = handler
15259            .handle_control(
15260                ConnectionId::new(2),
15261                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15262            )
15263            .unwrap();
15264        assert_eq!(wrong[0].header.ty, FrameType::Error);
15265        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15266        assert!(registry.get_module("vault").unwrap().is_none());
15267
15268        // A HELLO with the CORRECT nonce registers.
15269        let ok = handler
15270            .handle_control(
15271                ConnectionId::new(3),
15272                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15273            )
15274            .unwrap();
15275        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15276        assert!(registry.get_module("vault").unwrap().is_some());
15277    }
15278
15279    #[test]
15280    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15281        let registry = Arc::new(Registry::default());
15282        let supervisor = SupervisorHandle::new();
15283        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15284        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15285        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15286
15287        let squat = handler
15288            .handle_control(
15289                ConnectionId::new(1),
15290                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15291            )
15292            .unwrap();
15293        assert_eq!(squat[0].header.ty, FrameType::Error);
15294        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15295        assert!(parse_error(&squat[0])["message"]
15296            .as_str()
15297            .unwrap()
15298            .contains("fed:"));
15299
15300        let accepted_peer = handler
15301            .handle_control(
15302                ConnectionId::new(2),
15303                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15304            )
15305            .unwrap();
15306        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15307
15308        let accepted_short = handler
15309            .handle_control(
15310                ConnectionId::new(3),
15311                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15312            )
15313            .unwrap();
15314        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15315
15316        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15317            let response = handler
15318                .handle_control(
15319                    ConnectionId::new(conn),
15320                    hello_frame(module_id, PROTOCOL_VERSION, conn),
15321                )
15322                .unwrap();
15323            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15324        }
15325    }
15326
15327    #[test]
15328    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15329        let registry = Arc::new(Registry::default());
15330        let supervisor = SupervisorHandle::new();
15331        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15332        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15333        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15334        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15335
15336        let owner_nonce = handler
15337            .handle_control(
15338                ConnectionId::new(1),
15339                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15340            )
15341            .unwrap();
15342        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15343        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15344        assert!(registry.get_module("fed:special").unwrap().is_none());
15345
15346        let exact_nonce = handler
15347            .handle_control(
15348                ConnectionId::new(2),
15349                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15350            )
15351            .unwrap();
15352        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15353        assert!(registry.get_module("fed:special").unwrap().is_some());
15354    }
15355
15356    #[test]
15357    fn non_reserved_module_ignores_launch_nonce() {
15358        let registry = Arc::new(Registry::default());
15359        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
15360        // registration succeeds whether a spawned process echoes a nonce or not.
15361        let handler = ControlHandler::new(Arc::clone(&registry));
15362        let no_nonce = handler
15363            .handle_control(
15364                ConnectionId::new(1),
15365                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15366            )
15367            .unwrap();
15368        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15369        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15370
15371        let echoed_nonce = handler
15372            .handle_control(
15373                ConnectionId::new(2),
15374                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15375            )
15376            .unwrap();
15377        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15378        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15379    }
15380
15381    #[test]
15382    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15383        let handler = ControlHandler::default();
15384        let conn = ConnectionId::new(1);
15385        let malformed = Frame::build(
15386            FrameType::Hello,
15387            control_flags(),
15388            0,
15389            0,
15390            3,
15391            b"{not json".to_vec(),
15392        )
15393        .unwrap();
15394
15395        let error = handler.handle_control(conn, malformed).unwrap();
15396        assert_eq!(error[0].header.ty, FrameType::Error);
15397        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15398
15399        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15400        let pong = handler.handle_control(conn, ping).unwrap();
15401        assert_eq!(pong[0].header.ty, FrameType::Pong);
15402        assert_eq!(pong[0].header.corr, 4);
15403    }
15404
15405    #[test]
15406    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15407        let registry = Arc::new(Registry::default());
15408        let handler = ControlHandler::new(Arc::clone(&registry));
15409
15410        handler
15411            .handle_control(
15412                ConnectionId::new(1),
15413                hello_frame("aft", PROTOCOL_VERSION, 1),
15414            )
15415            .unwrap();
15416        let duplicate = handler
15417            .handle_control(
15418                ConnectionId::new(2),
15419                hello_frame("aft", PROTOCOL_VERSION, 2),
15420            )
15421            .unwrap();
15422
15423        assert_eq!(duplicate[0].header.ty, FrameType::Error);
15424        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15425        let registration = registry.get_module("aft").unwrap().unwrap();
15426        assert_eq!(registration.connection_id, ConnectionId::new(1));
15427    }
15428
15429    #[test]
15430    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15431        let registry = Arc::new(Registry::default());
15432        let forwarding = Arc::new(ForwardingTable::default());
15433        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15434        let handler =
15435            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15436                .with_process_liveness(process_liveness);
15437        let (ctx, route_channel, route_epoch) =
15438            bind_liveness_route(&registry, &forwarding, "aft-dead");
15439        let responses = handler
15440            .handle_route_poll(
15441                &ctx,
15442                route_poll_frame(41, PollKind::Liveness, route_channel),
15443                route_channel,
15444                route_epoch,
15445                PollKind::Liveness,
15446            )
15447            .unwrap();
15448
15449        assert_eq!(responses.len(), 1);
15450        assert_eq!(responses[0].header.ty, FrameType::Response);
15451        assert_route_poll_liveness(&responses[0], false);
15452    }
15453
15454    #[test]
15455    fn liveness_poll_without_process_source_uses_bound_route() {
15456        let registry = Arc::new(Registry::default());
15457        let forwarding = Arc::new(ForwardingTable::default());
15458        let handler =
15459            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15460        let (ctx, route_channel, route_epoch) =
15461            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
15462        let responses = handler
15463            .handle_route_poll(
15464                &ctx,
15465                route_poll_frame(42, PollKind::Liveness, route_channel),
15466                route_channel,
15467                route_epoch,
15468                PollKind::Liveness,
15469            )
15470            .unwrap();
15471
15472        assert_route_poll_liveness(&responses[0], true);
15473    }
15474
15475    #[test]
15476    fn liveness_poll_untracked_process_source_uses_bound_route() {
15477        let registry = Arc::new(Registry::default());
15478        let forwarding = Arc::new(ForwardingTable::default());
15479        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15480        let handler =
15481            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15482                .with_process_liveness(process_liveness);
15483        let (ctx, route_channel, route_epoch) =
15484            bind_liveness_route(&registry, &forwarding, "aft-untracked");
15485        let responses = handler
15486            .handle_route_poll(
15487                &ctx,
15488                route_poll_frame(43, PollKind::Liveness, route_channel),
15489                route_channel,
15490                route_epoch,
15491                PollKind::Liveness,
15492            )
15493            .unwrap();
15494
15495        assert_route_poll_liveness(&responses[0], true);
15496    }
15497
15498    #[tokio::test]
15499    async fn unknown_op_returns_unknown_control_op() {
15500        let handler = ControlHandler::default();
15501        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15502        let request = Frame::build(
15503            FrameType::Request,
15504            control_flags(),
15505            0,
15506            0,
15507            55,
15508            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15509        )
15510        .unwrap();
15511
15512        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15513
15514        assert_eq!(response.len(), 1);
15515        assert_eq!(response[0].header.ty, FrameType::Error);
15516        assert_eq!(response[0].header.corr, 55);
15517        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15518    }
15519
15520    #[tokio::test]
15521    async fn supervisor_provenance_rejects_unknown_exact_module() {
15522        let handler = ControlHandler::default();
15523        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15524        let request = Frame::build(
15525            FrameType::Request,
15526            control_flags(),
15527            0,
15528            0,
15529            57,
15530            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15531        )
15532        .unwrap();
15533
15534        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15535
15536        assert_eq!(response.len(), 1);
15537        assert_eq!(response[0].header.ty, FrameType::Error);
15538        assert_eq!(response[0].header.corr, 57);
15539        let error = parse_error(&response[0]);
15540        assert_eq!(error["code"], "unknown_module");
15541        assert_eq!(error["message"], "module_id 'missing' is not supervised");
15542    }
15543
15544    #[test]
15545    fn provenance_probe_override_keeps_handler_tests_deterministic() {
15546        let expected = subc_control::RunningImageAgreement::Unavailable {
15547            reason: subc_control::RunningImageUnavailableReason::HashFailed,
15548        };
15549        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15550        assert_eq!(handler.provenance_probe_override, Some(expected));
15551    }
15552
15553    #[test]
15554    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15555        let verdict = reload_verdict(
15556            std::path::Path::new("/bin/new"),
15557            Some(std::path::Path::new("/bin/old")),
15558            subc_control::RunningImageAgreement::Unavailable {
15559                reason: subc_control::RunningImageUnavailableReason::HashFailed,
15560            },
15561        );
15562        assert!(matches!(
15563            verdict.path,
15564            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15565                if configured == std::path::Path::new("/bin/new")
15566                    && spawned_from == std::path::Path::new("/bin/old")
15567        ));
15568    }
15569
15570    #[test]
15571    fn launch_spec_without_running_digest_is_unavailable_not_match() {
15572        use subc_control::{
15573            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15574        };
15575        let configured = crate::supervise::ModuleSpec {
15576            module_id: "legacy".into(),
15577            program: "/module".into(),
15578            args: Vec::new(),
15579            env: Vec::new(),
15580            reserved: false,
15581            reserved_prefixes: Vec::new(),
15582            protocol: subc_control::ModuleProtocol::Subc,
15583            overlap: Default::default(),
15584        };
15585        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15586        assert_eq!(
15587            launch_spec_agreement(&configured, false, Some(&digest)),
15588            Agreement::Unavailable {
15589                reason: Reason::NotRunning
15590            }
15591        );
15592        assert_eq!(
15593            launch_spec_agreement(&configured, true, None),
15594            Agreement::Unavailable {
15595                reason: Reason::SpawnedSpecUnavailable
15596            }
15597        );
15598        assert_eq!(
15599            launch_spec_agreement(&configured, true, Some(&digest)),
15600            Agreement::Match
15601        );
15602    }
15603
15604    #[cfg(windows)]
15605    #[test]
15606    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15607        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15608        let configured = dir.join("module.exe");
15609        std::fs::write(&configured, b"executable path fixture").unwrap();
15610        let spawned = std::fs::canonicalize(&configured).unwrap();
15611        assert_ne!(
15612            configured, spawned,
15613            "fixture must exercise different path spellings"
15614        );
15615        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15616        let lower_case = configured.to_string_lossy().to_lowercase();
15617        for spelling in [
15618            &configured,
15619            Path::new(&forward_slashes),
15620            Path::new(&lower_case),
15621        ] {
15622            let verdict = reload_verdict(
15623                spelling,
15624                Some(&spawned),
15625                subc_control::RunningImageAgreement::Unavailable {
15626                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15627                },
15628            );
15629            assert_eq!(
15630                verdict.path,
15631                ReloadPathAgreement::Match,
15632                "configured {spelling:?}, spawned {spawned:?}"
15633            );
15634        }
15635    }
15636
15637    #[cfg(windows)]
15638    #[test]
15639    fn windows_reload_path_keeps_different_existing_program_pending() {
15640        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15641        let old = dir.join("old.exe");
15642        let new = dir.join("new.exe");
15643        std::fs::write(&old, b"same executable content").unwrap();
15644        std::fs::copy(&old, &new).unwrap();
15645        let spawned = std::fs::canonicalize(&old).unwrap();
15646        let image = subc_control::RunningImageAgreement::Unavailable {
15647            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15648        };
15649        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15650        assert!(
15651            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15652            if configured == new && spawned_from == spawned)
15653        );
15654        let missing = dir.join("missing.exe");
15655        assert!(matches!(
15656            reload_verdict(&missing, Some(&spawned), image).path,
15657            ReloadPathAgreement::Mismatch { .. }
15658        ));
15659    }
15660
15661    #[test]
15662    fn reload_verdict_detects_replaced_image_at_same_path() {
15663        let image = subc_control::RunningImageAgreement::Mismatch {
15664            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15665                digest: "old".into(),
15666            },
15667            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15668                digest: "new".into(),
15669            },
15670        };
15671        let verdict = reload_verdict(
15672            std::path::Path::new("/bin/same"),
15673            Some(std::path::Path::new("/bin/same")),
15674            image.clone(),
15675        );
15676        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15677        assert_eq!(verdict.image, image);
15678    }
15679
15680    #[test]
15681    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15682        let image = subc_control::RunningImageAgreement::Unavailable {
15683            reason: subc_control::RunningImageUnavailableReason::NotRunning,
15684        };
15685        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15686        assert_eq!(
15687            verdict.path,
15688            subc_control::ReloadPathAgreement::Unavailable {
15689                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15690            }
15691        );
15692        assert_eq!(verdict.image, image);
15693
15694        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15695            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15696        };
15697        let verdict = reload_verdict(
15698            std::path::Path::new("/bin/same"),
15699            Some(std::path::Path::new("/bin/same")),
15700            unconfirmed.clone(),
15701        );
15702        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15703        assert_eq!(verdict.image, unconfirmed);
15704    }
15705
15706    #[test]
15707    fn reload_verdict_preserves_each_image_unavailability_reason() {
15708        use subc_control::RunningImageUnavailableReason as Reason;
15709
15710        for reason in [
15711            Reason::NotRunning,
15712            Reason::UnsupportedPlatform,
15713            Reason::RunningExecutableUnreadable,
15714            Reason::SpawnedPathUnreadable,
15715            Reason::HashFailed,
15716            Reason::ProcessIdentityUnconfirmed,
15717            Reason::Unknown("future_probe_reason".to_string()),
15718        ] {
15719            let image = subc_control::RunningImageAgreement::Unavailable {
15720                reason: reason.clone(),
15721            };
15722            let verdict = reload_verdict(
15723                std::path::Path::new("/bin/same"),
15724                Some(std::path::Path::new("/bin/same")),
15725                image.clone(),
15726            );
15727            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15728            assert_eq!(verdict.image, image, "{reason:?}");
15729        }
15730    }
15731
15732    #[tokio::test]
15733    async fn malformed_control_bodies_return_invalid_control_body() {
15734        let handler = ControlHandler::default();
15735        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15736
15737        for (corr, body) in [
15738            (56, br#"{"route_channel":1}"#.as_slice()),
15739            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15740            (
15741                58,
15742                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15743            ),
15744        ] {
15745            let request = Frame::build(
15746                FrameType::Request,
15747                control_flags(),
15748                0,
15749                0,
15750                corr,
15751                body.to_vec(),
15752            )
15753            .unwrap();
15754            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15755
15756            assert_eq!(response.len(), 1);
15757            assert_eq!(response[0].header.ty, FrameType::Error);
15758            assert_eq!(response[0].header.corr, corr);
15759            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15760        }
15761    }
15762
15763    #[tokio::test]
15764    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15765        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15766        let registry = Arc::new(Registry::default());
15767        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
15768        let router = Router::with_control_handler(Arc::clone(&control));
15769        let connection = router.begin_connection();
15770        let (ctx, mut rx) = route_ctx(connection.id());
15771
15772        router
15773            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15774            .await
15775            .unwrap();
15776        let response = rx.recv().await.unwrap();
15777        let ack = parse_ack(&response);
15778        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15779        let channel = 1;
15780
15781        let goodbye =
15782            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15783        router.route_for_connection(&ctx, goodbye).await.unwrap();
15784        assert!(rx.try_recv().is_err());
15785        assert!(registry.get_module("aft").unwrap().is_none());
15786
15787        router
15788            .route_for_connection(&ctx, channel_request(channel, 13))
15789            .await
15790            .unwrap();
15791        let error_frame = rx.recv().await.unwrap();
15792        assert_eq!(error_frame.header.ty, FrameType::Error);
15793        assert_eq!(error_frame.header.channel, channel);
15794        let captured = crate::router::test_log::captured_logs(&logs);
15795        assert_eq!(
15796            captured
15797                .lines()
15798                .filter(|line| {
15799                    line.contains("module_id=aft")
15800                        && line.contains("reason=explicit_goodbye")
15801                        && line.contains("module registration ended")
15802                })
15803                .count(),
15804            1,
15805            "unexpected GOODBYE registry log: {captured}"
15806        );
15807    }
15808
15809    #[tokio::test]
15810    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15811        let registry = Arc::new(Registry::default());
15812        let handler = ControlHandler::new(registry).with_capability_config(
15813            [("prov".to_string(), true), ("cons".to_string(), true)],
15814            BTreeMap::new(),
15815        );
15816        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15817        register_capability_manifest(
15818            &handler,
15819            &provider_ctx,
15820            &mut provider_rx,
15821            capability_manifest("prov", &["thing/v1"], &[]),
15822            1,
15823        )
15824        .await;
15825        let mut consumer = capability_manifest("cons", &[], &[]);
15826        consumer.capabilities.as_mut().unwrap().requires.push(
15827            subc_protocol::manifest::CapabilityRequirement {
15828                capability: "thing/v1".to_string(),
15829                need: subc_protocol::manifest::CapabilityNeed::Required,
15830            },
15831        );
15832        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15833        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15834        assert_eq!(
15835            handler.capability_evaluator.verdict("cons", "thing/v1"),
15836            Some(CapabilityVerdict::Provided)
15837        );
15838        let (mut client_rx, _) = open_route_for_capability_test(
15839            &handler,
15840            &provider_ctx,
15841            &mut provider_rx,
15842            703,
15843            3,
15844            "prov",
15845            None,
15846        )
15847        .await;
15848        let goodbye =
15849            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15850        handler
15851            .handle_control_frame(&provider_ctx, goodbye)
15852            .await
15853            .unwrap();
15854        assert_eq!(
15855            handler.capability_evaluator.verdict("cons", "thing/v1"),
15856            Some(CapabilityVerdict::NeverProvided)
15857        );
15858        let closed = client_rx
15859            .try_recv()
15860            .expect("GOODBYE pushes route.closed before route GOODBYE");
15861        assert!(
15862            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15863            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15864        );
15865        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15866        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15867    }
15868
15869    struct RouteClosedFixture {
15870        handler: ControlHandler,
15871        #[cfg(unix)]
15872        supervisor: Supervisor,
15873        module_ctx: RouteCtx,
15874        #[cfg(unix)]
15875        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15876        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15877        route: Arc<crate::forwarding::RouteBinding>,
15878    }
15879
15880    async fn route_closed_fixture() -> RouteClosedFixture {
15881        let registry = Arc::new(Registry::default());
15882        let handler = ControlHandler::new(Arc::clone(&registry));
15883        #[cfg(unix)]
15884        let supervisor = Supervisor::new(registry, RestartPolicy::default())
15885            .with_forwarding(Arc::clone(&handler.forwarding));
15886        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15887        register_capability_manifest(
15888            &handler,
15889            &module_ctx,
15890            &mut module_rx,
15891            capability_manifest("prov", &[], &[]),
15892            1,
15893        )
15894        .await;
15895        let (client_rx, bind) = open_route_for_capability_test(
15896            &handler,
15897            &module_ctx,
15898            &mut module_rx,
15899            702,
15900            2,
15901            "prov",
15902            None,
15903        )
15904        .await;
15905        let ModuleControlRequest::RouteBind {
15906            route_channel: channel,
15907            epoch: route_epoch,
15908            ..
15909        } = bind
15910        else {
15911            panic!("expected route.bind");
15912        };
15913        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15914            handler
15915                .forwarding
15916                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15917                .unwrap()
15918        else {
15919            panic!("the acknowledged route must be bound");
15920        };
15921        RouteClosedFixture {
15922            handler,
15923            #[cfg(unix)]
15924            supervisor,
15925            module_ctx,
15926            #[cfg(unix)]
15927            module_rx,
15928            client_rx,
15929            route,
15930        }
15931    }
15932
15933    impl RouteClosedFixture {
15934        #[cfg(unix)]
15935        async fn cut(&mut self) {
15936            let notices = async {
15937                let mut draining = self.module_rx.recv().await.unwrap();
15938                assert!(matches!(
15939                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15940                        &draining.body
15941                    )
15942                    .unwrap(),
15943                    subc_protocol::session::ModuleControlCommand::Draining {
15944                        reason: RouteCloseReason::Disable,
15945                        ..
15946                    }
15947                ));
15948                draining.flushed.take().unwrap().send(()).unwrap();
15949                let mut closing = self.client_rx.recv().await.unwrap();
15950                assert!(matches!(
15951                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15952                    ClientControlPush::RouteClosing {
15953                        reason: RouteCloseReason::Restart,
15954                        ..
15955                    }
15956                ));
15957                closing.flushed.take().unwrap().send(()).unwrap();
15958            };
15959            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15960            result.unwrap();
15961        }
15962
15963        fn disconnect(&mut self) -> ClientControlPush {
15964            self.handler
15965                .cleanup_connection(self.module_ctx.connection_id)
15966                .unwrap();
15967            let closed = self.client_rx.try_recv().unwrap();
15968            assert_eq!(closed.header.ty, FrameType::Push);
15969            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15970            assert!(matches!(
15971                &push,
15972                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15973                    if channels == &[self.route.client_channel]
15974            ));
15975            assert_eq!(
15976                self.client_rx.try_recv().unwrap().header.ty,
15977                FrameType::Goodbye
15978            );
15979            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15980            push
15981        }
15982    }
15983
15984    #[cfg(unix)]
15985    #[tokio::test(start_paused = true)]
15986    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15987        let mut fixture = route_closed_fixture().await;
15988        fixture.cut().await;
15989        assert!(matches!(
15990            fixture.disconnect(),
15991            ClientControlPush::RouteClosed {
15992                reason: RouteCloseReason::Restart,
15993                drained: true,
15994                ..
15995            }
15996        ));
15997    }
15998
15999    #[cfg(unix)]
16000    #[tokio::test(start_paused = true)]
16001    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16002        let mut fixture = route_closed_fixture().await;
16003        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16004        let started = tokio::time::Instant::now();
16005        fixture.cut().await;
16006        assert!(started.elapsed() >= Duration::from_secs(2));
16007        assert_eq!(fixture.route.flow.in_flight(), 1);
16008        assert!(matches!(
16009            fixture.disconnect(),
16010            ClientControlPush::RouteClosed {
16011                reason: RouteCloseReason::Restart,
16012                drained: false,
16013                ..
16014            }
16015        ));
16016    }
16017
16018    #[tokio::test]
16019    async fn module_connection_crash_reports_crash_and_not_drained() {
16020        let mut fixture = route_closed_fixture().await;
16021        // The server calls cleanup_connection on module EOF. Without a planned
16022        // daemon shutdown, that disconnect must remain a crash with no prior
16023        // route.closing notice, even when the route has no in-flight calls.
16024        assert!(matches!(
16025            fixture.disconnect(),
16026            ClientControlPush::RouteClosed {
16027                reason: RouteCloseReason::Crash,
16028                drained: false,
16029                ..
16030            }
16031        ));
16032    }
16033
16034    #[tokio::test]
16035    async fn dropping_router_connection_releases_registration() {
16036        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16037        let registry = Arc::new(Registry::default());
16038        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16039        let router = Router::with_control_handler(Arc::clone(&control));
16040        let connection = router.begin_connection();
16041        let connection_id = connection.id();
16042        let (ctx, mut rx) = route_ctx(connection_id);
16043
16044        router
16045            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16046            .await
16047            .unwrap();
16048        let response = rx.recv().await.unwrap();
16049        let ack = parse_ack(&response);
16050        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16051        assert!(registry.get_module("aft").unwrap().is_some());
16052
16053        drop(connection);
16054
16055        assert!(registry.get_module("aft").unwrap().is_none());
16056        assert_eq!(registry.active_registration_count().unwrap(), 0);
16057
16058        control
16059            .cleanup_connection(ConnectionId::new(u64::MAX))
16060            .unwrap();
16061        let captured = crate::router::test_log::captured_logs(&logs);
16062        println!("captured registry lifecycle logs:\n{captured}");
16063        let events: Vec<_> = captured
16064            .lines()
16065            .filter(|line| {
16066                line.contains("module registered module_id=aft ")
16067                    || line.contains("module registration ended")
16068            })
16069            .collect();
16070        assert_eq!(
16071            events.len(),
16072            2,
16073            "unexpected registry lifecycle logs: {captured}"
16074        );
16075        assert!(events[0].contains("module registered module_id=aft "));
16076        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16077        assert!(events[1].contains(&format!(
16078            "module_id=aft connection_id={}",
16079            connection_id.get()
16080        )));
16081        assert!(events[1].contains("reason=connection_closed"));
16082        assert!(events[1].contains("module registration ended"));
16083        assert_eq!(
16084            captured
16085                .lines()
16086                .filter(|line| line.contains("module registered module_id=aft "))
16087                .count(),
16088            1,
16089            "legacy registration admission line must appear once: {captured}"
16090        );
16091    }
16092
16093    #[tokio::test]
16094    async fn hello_registration_keeps_legacy_module_registered_line_once() {
16095        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16096        let registry = Arc::new(Registry::default());
16097        let control = ControlHandler::new(Arc::clone(&registry));
16098        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16099        hello_via_sink(
16100            &control,
16101            &ctx,
16102            &mut rx,
16103            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16104        )
16105        .await;
16106
16107        let captured = crate::router::test_log::captured_logs(&logs);
16108        let admissions: Vec<_> = captured
16109            .lines()
16110            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16111            .collect();
16112        assert_eq!(
16113            admissions.len(),
16114            1,
16115            "legacy admission line must remain exactly once: {captured}"
16116        );
16117        assert!(admissions[0].contains("routable_provider=true"));
16118        assert!(admissions[0].contains("connection_id=777"));
16119    }
16120
16121    fn capability_manifest(
16122        module_id: &str,
16123        provides: &[&str],
16124        must_never_reach: &[&str],
16125    ) -> ModuleManifest {
16126        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16127        manifest.capabilities = Some(CapabilityDeclarations {
16128            provides: provides
16129                .iter()
16130                .map(|capability| (*capability).to_string())
16131                .collect(),
16132            requires: Vec::new(),
16133            must_never_reach: must_never_reach
16134                .iter()
16135                .map(|capability| (*capability).to_string())
16136                .collect(),
16137        });
16138        manifest
16139    }
16140
16141    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16142        Frame::build(
16143            FrameType::Hello,
16144            control_flags(),
16145            0,
16146            0,
16147            corr,
16148            serde_json::to_vec(&ModuleHelloBody {
16149                protocol_ver: manifest.protocol_ver,
16150                manifest,
16151                control_ops: None,
16152                launch_nonce: None,
16153            })
16154            .expect("capability test HELLO serializes"),
16155        )
16156        .expect("capability test HELLO frame builds")
16157    }
16158
16159    fn catalog_update_with_capabilities_frame(
16160        corr: u64,
16161        capabilities: CapabilityDeclarations,
16162    ) -> Frame {
16163        Frame::build(
16164            FrameType::Request,
16165            control_flags(),
16166            0,
16167            0,
16168            corr,
16169            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16170                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16171                capabilities: Some(capabilities),
16172                ready: None,
16173            })
16174            .expect("capability catalog.update serializes"),
16175        )
16176        .expect("capability catalog.update frame builds")
16177    }
16178
16179    async fn register_capability_manifest(
16180        handler: &ControlHandler,
16181        ctx: &RouteCtx,
16182        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16183        manifest: ModuleManifest,
16184        corr: u64,
16185    ) {
16186        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16187    }
16188
16189    async fn open_route_for_capability_test(
16190        handler: &ControlHandler,
16191        target_ctx: &RouteCtx,
16192        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16193        client_connection_id: u64,
16194        corr: u64,
16195        target_module_id: &str,
16196        consumer_identity: Option<ConsumerIdentity>,
16197    ) -> (
16198        mpsc::Receiver<crate::router::OutboundFrame>,
16199        ModuleControlRequest,
16200    ) {
16201        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16202        let route_handler = handler.clone();
16203        let target_module_id = target_module_id.to_string();
16204        let route_task = tokio::spawn(async move {
16205            route_handler
16206                .handle_control_frame(
16207                    &client_ctx,
16208                    route_open_frame_with_admission_facts(
16209                        corr,
16210                        &target_module_id,
16211                        unique_project_root("admission-facts"),
16212                        consumer_identity,
16213                        None,
16214                    ),
16215                )
16216                .await
16217                .expect("capability test route.open succeeds")
16218        });
16219        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16220            .await
16221            .expect("capability test route.open must reach route.bind")
16222            .expect("target control receiver stays open");
16223        let bind_request: ModuleControlRequest =
16224            serde_json::from_slice(&bind.body).expect("route.bind decodes");
16225        handler
16226            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16227            .await
16228            .expect("capability test route.bind ACK succeeds");
16229        assert!(route_task.await.expect("route.open task joins").is_empty());
16230        let opened = client_rx
16231            .recv()
16232            .await
16233            .expect("successful route.open publishes a response");
16234        assert!(matches!(
16235            serde_json::from_slice::<ClientControlResponse>(&opened.body),
16236            Ok(ClientControlResponse::RouteOpen { .. })
16237        ));
16238        (client_rx, bind_request)
16239    }
16240
16241    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16242        assert_eq!(frame.header.ty, FrameType::Push);
16243        assert_eq!(frame.header.channel, 0);
16244        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16245            .expect("route.closed control push decodes");
16246        let ClientControlPush::RouteClosed { channels, .. } = &push else {
16247            panic!("expected route.closed");
16248        };
16249        assert_eq!(channels.len(), 1, "exactly one violating route closed");
16250        let closed_channel = channels[0];
16251        let channels = channels.clone();
16252        assert_eq!(
16253            push,
16254            ClientControlPush::RouteClosed {
16255                module_id: target_module_id.to_string(),
16256                channels,
16257                reason: RouteCloseReason::CapabilityDenied,
16258                drained: false,
16259                abandoned: 0,
16260                excluded_subscriptions: 0,
16261                terminal: Some(false),
16262            }
16263        );
16264        closed_channel
16265    }
16266
16267    #[tokio::test]
16268    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16269        let registry = Arc::new(Registry::default());
16270        let forwarding = Arc::new(ForwardingTable::default());
16271        let supervisor = SupervisorHandle::new();
16272        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16273        let handler =
16274            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16275                .with_supervisor(supervisor);
16276        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16277        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16278        register_capability_manifest(
16279            &handler,
16280            &target_ctx,
16281            &mut target_rx,
16282            capability_manifest("target", &["credentials-provider/v1"], &[]),
16283            1,
16284        )
16285        .await;
16286        register_capability_manifest(
16287            &handler,
16288            &opener_ctx,
16289            &mut opener_rx,
16290            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16291            2,
16292        )
16293        .await;
16294
16295        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16296        let replies = handler
16297            .handle_control_frame(
16298                &client_ctx,
16299                route_open_frame_with_admission_facts(
16300                    3,
16301                    "target",
16302                    unique_project_root("admission-facts"),
16303                    Some(ConsumerIdentity {
16304                        module_id: "opener".to_string(),
16305                        launch_nonce: "opener-nonce".to_string(),
16306                    }),
16307                    None,
16308                ),
16309            )
16310            .await
16311            .expect("denied route.open returns a typed frame");
16312        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16313        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16314        assert!(
16315            target_rx.try_recv().is_err(),
16316            "forbidden route.open must not relay route.bind"
16317        );
16318    }
16319
16320    #[tokio::test]
16321    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16322        let registry = Arc::new(Registry::default());
16323        let forwarding = Arc::new(ForwardingTable::default());
16324        let supervisor = SupervisorHandle::new();
16325        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16326        let handler =
16327            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16328                .with_supervisor(supervisor);
16329        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16330        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16331        register_capability_manifest(
16332            &handler,
16333            &target_ctx,
16334            &mut target_rx,
16335            capability_manifest("target", &["credentials-provider/v1"], &[]),
16336            1,
16337        )
16338        .await;
16339        register_capability_manifest(
16340            &handler,
16341            &old_opener_ctx,
16342            &mut old_opener_rx,
16343            capability_manifest("opener", &[], &[]),
16344            2,
16345        )
16346        .await;
16347        let (mut client_rx, _) = open_route_for_capability_test(
16348            &handler,
16349            &target_ctx,
16350            &mut target_rx,
16351            712,
16352            3,
16353            "target",
16354            Some(ConsumerIdentity {
16355                module_id: "opener".to_string(),
16356                launch_nonce: "opener-nonce".to_string(),
16357            }),
16358        )
16359        .await;
16360        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16361
16362        handler
16363            .cleanup_connection(old_opener_ctx.connection_id)
16364            .expect("old opener registration cleans up");
16365        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16366        register_capability_manifest(
16367            &handler,
16368            &new_opener_ctx,
16369            &mut new_opener_rx,
16370            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16371            4,
16372        )
16373        .await;
16374
16375        let closed_channel = assert_capability_denied_push(
16376            client_rx
16377                .try_recv()
16378                .expect("HELLO deny addition must emit route.closed")
16379                .frame,
16380            "target",
16381        );
16382        let goodbye = client_rx
16383            .try_recv()
16384            .expect("HELLO deny addition must send the client a route GOODBYE")
16385            .frame;
16386        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16387        assert_eq!(goodbye.header.channel, closed_channel);
16388        assert_ne!(goodbye.header.channel, 0);
16389        assert_ne!(goodbye.header.epoch, 0);
16390        assert!(client_rx.try_recv().is_err());
16391        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16392        assert!(matches!(
16393            target_rx.try_recv(),
16394            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16395        ));
16396    }
16397
16398    #[tokio::test]
16399    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16400        let registry = Arc::new(Registry::default());
16401        let forwarding = Arc::new(ForwardingTable::default());
16402        let supervisor = SupervisorHandle::new();
16403        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16404        let handler =
16405            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16406                .with_supervisor(supervisor);
16407        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16408        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16409        register_capability_manifest(
16410            &handler,
16411            &target_ctx,
16412            &mut target_rx,
16413            capability_manifest("target", &[], &[]),
16414            1,
16415        )
16416        .await;
16417        register_capability_manifest(
16418            &handler,
16419            &opener_ctx,
16420            &mut opener_rx,
16421            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16422            2,
16423        )
16424        .await;
16425        let (mut client_rx, _) = open_route_for_capability_test(
16426            &handler,
16427            &target_ctx,
16428            &mut target_rx,
16429            722,
16430            3,
16431            "target",
16432            Some(ConsumerIdentity {
16433                module_id: "opener".to_string(),
16434                launch_nonce: "opener-nonce".to_string(),
16435            }),
16436        )
16437        .await;
16438        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16439
16440        let replies = handler
16441            .handle_control_frame(
16442                &target_ctx,
16443                catalog_update_with_capabilities_frame(
16444                    4,
16445                    CapabilityDeclarations {
16446                        provides: vec!["credentials-provider/v1".to_string()],
16447                        requires: Vec::new(),
16448                        must_never_reach: Vec::new(),
16449                    },
16450                ),
16451            )
16452            .await
16453            .expect("claim catalog.update succeeds");
16454        assert!(matches!(
16455            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16456            Ok(ModuleControlResponseToModule::CatalogUpdate {})
16457        ));
16458        let closed_channel = assert_capability_denied_push(
16459            client_rx
16460                .try_recv()
16461                .expect("claim addition must emit route.closed")
16462                .frame,
16463            "target",
16464        );
16465        let goodbye = client_rx
16466            .try_recv()
16467            .expect("claim addition must send the client a route GOODBYE")
16468            .frame;
16469        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16470        assert_eq!(goodbye.header.channel, closed_channel);
16471        assert_ne!(goodbye.header.channel, 0);
16472        assert_ne!(goodbye.header.epoch, 0);
16473        assert!(client_rx.try_recv().is_err());
16474        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16475        assert!(matches!(
16476            target_rx.try_recv(),
16477            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16478        ));
16479    }
16480
16481    #[tokio::test]
16482    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16483        let registry = Arc::new(Registry::default());
16484        let forwarding = Arc::new(ForwardingTable::default());
16485        let supervisor = SupervisorHandle::new();
16486        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16487        let handler =
16488            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16489                .with_supervisor(supervisor);
16490        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16491        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16492        register_capability_manifest(
16493            &handler,
16494            &target_ctx,
16495            &mut target_rx,
16496            capability_manifest("target", &["credentials-provider/v1"], &[]),
16497            1,
16498        )
16499        .await;
16500        register_capability_manifest(
16501            &handler,
16502            &opener_ctx,
16503            &mut opener_rx,
16504            capability_manifest("opener", &[], &[]),
16505            2,
16506        )
16507        .await;
16508        let (mut client_rx, _) = open_route_for_capability_test(
16509            &handler,
16510            &target_ctx,
16511            &mut target_rx,
16512            732,
16513            3,
16514            "target",
16515            Some(ConsumerIdentity {
16516                module_id: "opener".to_string(),
16517                launch_nonce: "opener-nonce".to_string(),
16518            }),
16519        )
16520        .await;
16521        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16522
16523        handler
16524            .handle_control_frame(
16525                &target_ctx,
16526                catalog_update_with_capabilities_frame(
16527                    4,
16528                    CapabilityDeclarations {
16529                        provides: Vec::new(),
16530                        requires: Vec::new(),
16531                        must_never_reach: Vec::new(),
16532                    },
16533                ),
16534            )
16535            .await
16536            .expect("claim removal catalog.update succeeds");
16537        assert_eq!(
16538            forwarding.active_binding_count().unwrap(),
16539            1,
16540            "removing an attested target claim must leave the route census unchanged"
16541        );
16542        assert!(
16543            client_rx.try_recv().is_err(),
16544            "claim removal must not emit route.closed or a client route GOODBYE"
16545        );
16546        assert!(
16547            target_rx.try_recv().is_err(),
16548            "claim removal must not send the target a route GOODBYE"
16549        );
16550    }
16551
16552    /// A direct client may open a route to a denied capability provider; this
16553    /// policy applies only to attested supervised module origins, not to direct clients.
16554    #[tokio::test]
16555    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16556        let registry = Arc::new(Registry::default());
16557        let forwarding = Arc::new(ForwardingTable::default());
16558        let supervisor = SupervisorHandle::new();
16559        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16560        let handler =
16561            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16562                .with_supervisor(supervisor);
16563        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16564        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16565        register_capability_manifest(
16566            &handler,
16567            &target_ctx,
16568            &mut target_rx,
16569            capability_manifest("target", &["credentials-provider/v1"], &[]),
16570            1,
16571        )
16572        .await;
16573        register_capability_manifest(
16574            &handler,
16575            &opener_ctx,
16576            &mut opener_rx,
16577            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16578            2,
16579        )
16580        .await;
16581
16582        let (_client_rx, bind) = open_route_for_capability_test(
16583            &handler,
16584            &target_ctx,
16585            &mut target_rx,
16586            742,
16587            3,
16588            "target",
16589            None,
16590        )
16591        .await;
16592        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16593            panic!("direct scope-honesty route must bind");
16594        };
16595        assert_eq!(principal, Some(Principal::Direct));
16596        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16597    }
16598
16599    /// A module that denies a capability receives no self-route exemption when it
16600    /// also attestedly provides that capability.
16601    #[tokio::test]
16602    async fn must_never_reach_self_route_is_capability_forbidden() {
16603        let registry = Arc::new(Registry::default());
16604        let forwarding = Arc::new(ForwardingTable::default());
16605        let supervisor = SupervisorHandle::new();
16606        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16607        let handler =
16608            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16609                .with_supervisor(supervisor);
16610        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16611        register_capability_manifest(
16612            &handler,
16613            &self_ctx,
16614            &mut self_rx,
16615            capability_manifest(
16616                "self-provider",
16617                &["credentials-provider/v1"],
16618                &["credentials-provider/v1"],
16619            ),
16620            1,
16621        )
16622        .await;
16623
16624        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16625        let replies = handler
16626            .handle_control_frame(
16627                &client_ctx,
16628                route_open_frame_with_admission_facts(
16629                    2,
16630                    "self-provider",
16631                    unique_project_root("admission-facts"),
16632                    Some(ConsumerIdentity {
16633                        module_id: "self-provider".to_string(),
16634                        launch_nonce: "self-nonce".to_string(),
16635                    }),
16636                    None,
16637                ),
16638            )
16639            .await
16640            .expect("self-route refusal returns a typed frame");
16641        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16642        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16643        assert!(
16644            self_rx.try_recv().is_err(),
16645            "self denial must not relay route.bind"
16646        );
16647    }
16648
16649    #[test]
16650    fn unsupported_channel_zero_frame_returns_error() {
16651        let handler = ControlHandler::default();
16652        let request = Frame::build(
16653            FrameType::Request,
16654            control_flags(),
16655            0,
16656            0,
16657            21,
16658            b"opaque".to_vec(),
16659        )
16660        .unwrap();
16661
16662        let response = handler
16663            .handle_control(ConnectionId::new(1), request)
16664            .unwrap();
16665
16666        assert_eq!(response[0].header.ty, FrameType::Error);
16667        assert_eq!(
16668            parse_error(&response[0])["code"],
16669            "unsupported_control_frame"
16670        );
16671    }
16672
16673    /// Blue/green swap at the control-plane boundary. The supervisor that opens
16674    /// a swap is not wired yet, so the candidate is registered here directly
16675    /// into the registry and forwarding candidate slots, the way the swap's
16676    /// HELLO admission will.
16677    mod swap {
16678        use super::*;
16679
16680        const INCUMBENT: ConnectionId = ConnectionId::new(30);
16681        const CANDIDATE: ConnectionId = ConnectionId::new(40);
16682
16683        struct Swap {
16684            registry: Arc<Registry>,
16685            forwarding: Arc<ForwardingTable>,
16686            handler: ControlHandler,
16687            incumbent_ctx: RouteCtx,
16688            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16689            candidate_ctx: RouteCtx,
16690            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16691        }
16692
16693        async fn swap_with_incumbent() -> Swap {
16694            let registry = Arc::new(Registry::default());
16695            let forwarding = Arc::new(ForwardingTable::default());
16696            let handler =
16697                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
16698            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16699            hello_via_sink(
16700                &handler,
16701                &incumbent_ctx,
16702                &mut incumbent_rx,
16703                hello_frame("aft", PROTOCOL_VERSION, 7),
16704            )
16705            .await;
16706            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16707            Swap {
16708                registry,
16709                forwarding,
16710                handler,
16711                incumbent_ctx,
16712                incumbent_rx,
16713                candidate_ctx,
16714                candidate_rx,
16715            }
16716        }
16717
16718        fn register_candidate(swap: &Swap, ready: Option<bool>) {
16719            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16720            candidate_manifest.ready = ready;
16721            let registration = swap
16722                .registry
16723                .register_candidate_with_control_ops(
16724                    candidate_manifest,
16725                    PROTOCOL_VERSION,
16726                    CANDIDATE,
16727                    module_baseline_control_ops(),
16728                )
16729                .unwrap();
16730            swap.forwarding
16731                .register_candidate_module_connection(
16732                    CANDIDATE,
16733                    "aft".to_string(),
16734                    PROTOCOL_VERSION,
16735                    manifest_concurrency(&registration.manifest),
16736                    swap.candidate_ctx.egress.clone(),
16737                )
16738                .unwrap();
16739        }
16740
16741        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16742            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16743            swap.registry.promote_candidate("aft").unwrap().unwrap();
16744            cutover.incumbent.unwrap()
16745        }
16746
16747        /// A swap candidate's HELLO reserves its candidate registry entry
16748        /// before installing its candidate forwarding endpoint. The swap
16749        /// supervisor promotes a ready candidate as soon as it reads it as
16750        /// registered, and promotion cuts forwarding over to the candidate
16751        /// endpoint, so the lookup the supervisor reads
16752        /// (`Registry::published_registration`) must not return the candidate
16753        /// until that endpoint exists. Otherwise the cutover finds no candidate
16754        /// endpoint and the swap fails as if the candidate had disconnected.
16755        #[tokio::test]
16756        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16757            let mut swap = swap_with_incumbent().await;
16758            let hold = Arc::new(std::sync::Barrier::new(2));
16759            let handler = swap
16760                .handler
16761                .clone()
16762                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16763            handler
16764                .supervisor
16765                .open_swap("aft", "candidate-nonce".to_string());
16766            let hello = {
16767                let handler = handler.clone();
16768                let candidate_ctx = swap.candidate_ctx.clone();
16769                std::thread::spawn(move || {
16770                    handler
16771                        .handle_hello(
16772                            candidate_ctx.connection_id,
16773                            Some(candidate_ctx.egress.clone()),
16774                            hello_frame_with_nonce(
16775                                "aft",
16776                                PROTOCOL_VERSION,
16777                                8,
16778                                Some("candidate-nonce"),
16779                            ),
16780                        )
16781                        .unwrap()
16782                })
16783            };
16784
16785            hold.wait();
16786            // Parked: the candidate slot is claimed but not published, and
16787            // forwarding has no candidate endpoint.
16788            assert!(swap
16789                .registry
16790                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16791                .unwrap()
16792                .is_some());
16793            assert!(swap
16794                .registry
16795                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16796                .unwrap()
16797                .is_none());
16798            assert!(swap
16799                .forwarding
16800                .module_endpoint_for_connection(CANDIDATE)
16801                .unwrap()
16802                .is_none());
16803
16804            hold.wait();
16805            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16806            assert_eq!(
16807                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16808                FrameType::HelloAck
16809            );
16810            let published = swap
16811                .registry
16812                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16813                .unwrap()
16814                .expect("candidate is published once its HELLO completes");
16815            assert_eq!(published.connection_id, CANDIDATE);
16816            assert!(swap
16817                .forwarding
16818                .module_endpoint_for_connection(CANDIDATE)
16819                .unwrap()
16820                .is_some());
16821        }
16822
16823        fn keyed_total(counters: &Value, key: &str) -> u64 {
16824            counters[key]
16825                .as_object()
16826                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16827                .unwrap_or(0)
16828        }
16829
16830        #[tokio::test]
16831        async fn replacement_logs_old_end_and_new_admission_once() {
16832            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16833            let mut swap = swap_with_incumbent().await;
16834            swap.handler
16835                .supervisor
16836                .open_swap("aft", "candidate-nonce".to_string());
16837            hello_via_sink(
16838                &swap.handler,
16839                &swap.candidate_ctx,
16840                &mut swap.candidate_rx,
16841                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16842            )
16843            .await;
16844            cutover(&swap);
16845            swap.handler.cleanup_connection(INCUMBENT).unwrap();
16846
16847            let captured = crate::router::test_log::captured_logs(&logs);
16848            println!("captured replacement registry logs:\n{captured}");
16849            let new_admissions: Vec<_> = captured
16850                .lines()
16851                .filter(|line| {
16852                    line.contains("connection_id=40")
16853                        && line.contains("swap candidate registered; not routable until cutover")
16854                })
16855                .collect();
16856            assert_eq!(
16857                new_admissions.len(),
16858                1,
16859                "unexpected admission logs: {captured}"
16860            );
16861            assert!(new_admissions[0].contains("module_id=aft"));
16862            assert!(new_admissions[0].contains("connection_id=40"));
16863            assert!(
16864                new_admissions[0].contains("swap candidate registered; not routable until cutover")
16865            );
16866            assert!(new_admissions[0].contains("ready=true"));
16867
16868            let old_admissions: Vec<_> = captured
16869                .lines()
16870                .filter(|line| {
16871                    line.contains("module registered module_id=aft ")
16872                        && line.contains("connection_id=30")
16873                })
16874                .collect();
16875            assert_eq!(
16876                old_admissions.len(),
16877                1,
16878                "unexpected incumbent admission: {captured}"
16879            );
16880
16881            let old_ends: Vec<_> = captured
16882                .lines()
16883                .filter(|line| {
16884                    line.contains("connection_id=30") && line.contains("module registration ended")
16885                })
16886                .collect();
16887            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16888            assert!(old_ends[0].contains("module_id=aft"));
16889            assert!(old_ends[0].contains("connection_id=30"));
16890            assert!(old_ends[0].contains("reason=replaced"));
16891            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16892            assert_eq!(
16893                captured
16894                    .lines()
16895                    .filter(|line| line.contains("module registration promoted"))
16896                    .count(),
16897                1,
16898                "unexpected promotion logs: {captured}"
16899            );
16900        }
16901
16902        /// An ack from the incumbent for a bind it was sent before cutover,
16903        /// arriving before the incumbent is drained. The incumbent is the live
16904        /// connection carrying every other client's routes, so the ack must
16905        /// not end it: the waiting client is told to retry, the reservation is
16906        /// given back, and the incumbent is told to drop just that binding.
16907        #[tokio::test]
16908        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16909            let mut swap = swap_with_incumbent().await;
16910            let handler = swap.handler.clone();
16911
16912            // A co-tenant route, bound on the incumbent before the swap.
16913            let cotenant = ConnectionId::new(31);
16914            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16915            let (cotenant_task, cotenant_bind) = relay_route_open(
16916                &handler,
16917                cotenant,
16918                &cotenant_ctx.egress,
16919                &mut swap.incumbent_rx,
16920                100,
16921                "aft",
16922                "swap-cotenant",
16923            )
16924            .await;
16925            handler
16926                .handle_control_frame(
16927                    &swap.incumbent_ctx,
16928                    route_bind_ack(cotenant_bind.header.corr),
16929                )
16930                .await
16931                .unwrap();
16932            assert!(cotenant_task.await.unwrap().is_empty());
16933            let (cotenant_channel, cotenant_epoch) =
16934                published_route(&cotenant_rx.recv().await.unwrap());
16935
16936            // A second route.open, relayed to the incumbent and not yet acked.
16937            let caller = ConnectionId::new(32);
16938            let (caller_ctx, mut caller_rx) = route_ctx(caller);
16939            let (caller_task, caller_bind) = relay_route_open(
16940                &handler,
16941                caller,
16942                &caller_ctx.egress,
16943                &mut swap.incumbent_rx,
16944                101,
16945                "aft",
16946                "swap-caller",
16947            )
16948            .await;
16949            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16950
16951            register_candidate(&swap, None);
16952            cutover(&swap);
16953
16954            // The incumbent acks after promotion and before any drain.
16955            let ack = handler
16956                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16957                .await;
16958            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16959            if module_loop_error.is_some() {
16960                // What the connection loop does with an untranslated router
16961                // error: end the connection, releasing every route on it.
16962                handler.cleanup_connection(INCUMBENT).unwrap();
16963            }
16964
16965            // 1. The incumbent's other routes survive.
16966            assert!(
16967                cotenant_rx.try_recv().is_err(),
16968                "the co-tenant route on the incumbent was torn down by one late ack: \
16969                 {module_loop_error:?}"
16970            );
16971            assert!(matches!(
16972                swap.forwarding
16973                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16974                    .unwrap(),
16975                DataRoute::Client(DataRouteState::Bound(_))
16976            ));
16977            assert_eq!(module_loop_error, None);
16978            assert!(swap
16979                .registry
16980                .get_module_by_connection(INCUMBENT)
16981                .unwrap()
16982                .is_some());
16983
16984            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
16985            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16986                .await
16987                .expect("the incumbent is told to drop the abandoned binding")
16988                .unwrap()
16989                .frame;
16990            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16991            assert_eq!(goodbye.header.channel, abandoned_channel);
16992            assert_eq!(goodbye.header.epoch, abandoned_epoch);
16993            assert!(swap.incumbent_rx.try_recv().is_err());
16994
16995            // 3. The waiting client gets a retryable refusal and no route.
16996            let response = caller_task.await.unwrap();
16997            assert_eq!(response.len(), 1);
16998            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16999            assert!(caller_rx.try_recv().is_err());
17000
17001            // 4. The reservation pair is given back, and the pending bind
17002            //    settled exactly once: one accepted open (the co-tenant) and one
17003            //    refused open (the caller), nothing counted twice.
17004            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17005            let counters = handler.counters().snapshot();
17006            assert_eq!(
17007                keyed_total(&counters, "route_open_accepted_by_principal"),
17008                1
17009            );
17010            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17011            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17012        }
17013
17014        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
17015        /// id would resolve to the promoted candidate and every new route.open
17016        /// would be refused as reloading, leaving neither process routable.
17017        #[tokio::test]
17018        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17019            let mut swap = swap_with_incumbent().await;
17020            register_candidate(&swap, None);
17021            let incumbent = cutover(&swap);
17022            swap.forwarding
17023                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17024                .unwrap()
17025                .expect("the incumbent is still registered");
17026
17027            let client = ConnectionId::new(33);
17028            let (client_ctx, mut client_rx) = route_ctx(client);
17029            let route_handler = swap.handler.clone();
17030            let open_ctx = RouteCtx {
17031                connection_id: client,
17032                egress: client_ctx.egress.clone(),
17033            };
17034            let mut route_task = tokio::spawn(async move {
17035                route_handler
17036                    .handle_control_frame(
17037                        &open_ctx,
17038                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17039                    )
17040                    .await
17041                    .unwrap()
17042            });
17043            let bind = tokio::select! {
17044                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17045                response = &mut route_task => {
17046                    let response = response.unwrap();
17047                    panic!(
17048                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
17049                        parse_error(&response[0])["code"]
17050                    );
17051                }
17052            };
17053            swap.handler
17054                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17055                .await
17056                .unwrap();
17057            assert!(route_task.await.unwrap().is_empty());
17058            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17059            match swap
17060                .forwarding
17061                .lookup_data_route(client, channel, epoch)
17062                .unwrap()
17063            {
17064                DataRoute::Client(DataRouteState::Bound(route)) => {
17065                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17066                }
17067                other => panic!("expected a bound route on the candidate, got {other:?}"),
17068            }
17069            assert!(swap.incumbent_rx.try_recv().is_err());
17070        }
17071
17072        /// A candidate declares itself ready with `catalog.update` on its own
17073        /// connection. If the connection-keyed registry lookups searched only the
17074        /// active slot, this would answer `not_registered` and the candidate
17075        /// would never become ready.
17076        #[tokio::test]
17077        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17078            let swap = swap_with_incumbent().await;
17079            register_candidate(&swap, Some(false));
17080            let update = Frame::build(
17081                FrameType::Request,
17082                control_flags(),
17083                0,
17084                0,
17085                55,
17086                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17087                    provides: manifest("aft", PROTOCOL_VERSION).provides,
17088                    capabilities: None,
17089                    ready: Some(true),
17090                })
17091                .unwrap(),
17092            )
17093            .unwrap();
17094
17095            let replies = swap
17096                .handler
17097                .handle_control_frame(&swap.candidate_ctx, update)
17098                .await
17099                .unwrap();
17100
17101            assert_eq!(replies.len(), 1);
17102            assert_eq!(
17103                replies[0].header.ty,
17104                FrameType::Response,
17105                "candidate catalog.update was refused: {:?}",
17106                serde_json::from_slice::<Value>(&replies[0].body).ok()
17107            );
17108            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17109            assert_eq!(
17110                swap.registry
17111                    .get_module("aft")
17112                    .unwrap()
17113                    .unwrap()
17114                    .connection_id,
17115                INCUMBENT
17116            );
17117        }
17118    }
17119
17120    /// The HELLO gate while the supervisor has a swap open: only the nonce it
17121    /// minted for the candidate admits a second process, into the candidate
17122    /// slot, and that check runs ahead of the reserved-module gate.
17123    mod swap_admission {
17124        use super::*;
17125
17126        const INCUMBENT_NONCE: &str = "incumbent-nonce";
17127        const CANDIDATE_NONCE: &str = "candidate-nonce";
17128
17129        fn handler_with_incumbent(
17130            module_id: &str,
17131            reserved: bool,
17132        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17133            let registry = Arc::new(Registry::default());
17134            let supervisor = SupervisorHandle::new();
17135            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17136            if reserved {
17137                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17138            }
17139            let handler =
17140                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
17141            let incumbent = handler
17142                .handle_control(
17143                    ConnectionId::new(1),
17144                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17145                )
17146                .unwrap();
17147            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17148            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17149            (registry, supervisor, handler)
17150        }
17151
17152        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
17153        /// admits every nonce, so while a swap is open the swap gate is the only
17154        /// thing between a key-holder and the candidate slot. A nonce the
17155        /// supervisor did not mint, or none at all, is refused, and neither the
17156        /// incumbent's registration nor the candidate slot moves.
17157        #[test]
17158        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17159            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17160
17161            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17162                let replies = handler
17163                    .handle_control(
17164                        ConnectionId::new(connection),
17165                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17166                    )
17167                    .unwrap();
17168                assert_eq!(replies[0].header.ty, FrameType::Error);
17169                assert_eq!(
17170                    parse_error(&replies[0])["code"],
17171                    "swap_token_invalid",
17172                    "nonce {nonce:?}"
17173                );
17174            }
17175            assert!(registry.get_candidate("aft").unwrap().is_none());
17176            assert_eq!(
17177                registry.get_module("aft").unwrap().unwrap().connection_id,
17178                ConnectionId::new(1)
17179            );
17180
17181            // Control: the minted token is admitted, into the candidate slot,
17182            // and only once.
17183            let admitted = handler
17184                .handle_control(
17185                    ConnectionId::new(4),
17186                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17187                )
17188                .unwrap();
17189            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17190            assert_eq!(
17191                registry
17192                    .get_candidate("aft")
17193                    .unwrap()
17194                    .unwrap()
17195                    .connection_id,
17196                ConnectionId::new(4)
17197            );
17198            assert_eq!(
17199                registry.get_module("aft").unwrap().unwrap().connection_id,
17200                ConnectionId::new(1),
17201                "the candidate must not take the active slot"
17202            );
17203            let replayed = handler
17204                .handle_control(
17205                    ConnectionId::new(5),
17206                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17207                )
17208                .unwrap();
17209            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17210
17211            // The case only this gate covers: the incumbent has died mid-swap,
17212            // so its duplicate refusal is gone too, and without the gate a
17213            // key-holder would take the id's ACTIVE slot.
17214            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17215            let squatter = handler
17216                .handle_control(
17217                    ConnectionId::new(6),
17218                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17219                )
17220                .unwrap();
17221            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17222            assert!(
17223                registry.get_module("aft").unwrap().is_none(),
17224                "a squatter took the active slot of an id being swapped"
17225            );
17226        }
17227
17228        /// Design mutation arm (iii). A reserved module's candidate presents a
17229        /// nonce the reserved gate has never seen (that gate holds the
17230        /// incumbent's), so the swap gate must run first or the candidate is
17231        /// refused `reserved_module` and a reserved module can never be swapped.
17232        #[test]
17233        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17234            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17235
17236            let replies = handler
17237                .handle_control(
17238                    ConnectionId::new(2),
17239                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17240                )
17241                .unwrap();
17242
17243            assert_eq!(
17244                replies[0].header.ty,
17245                FrameType::HelloAck,
17246                "reserved candidate refused: {:?}",
17247                serde_json::from_slice::<Value>(&replies[0].body).ok()
17248            );
17249            assert_eq!(
17250                registry
17251                    .get_candidate("vault")
17252                    .unwrap()
17253                    .unwrap()
17254                    .connection_id,
17255                ConnectionId::new(2)
17256            );
17257        }
17258
17259        /// With no swap open the gate is inert: the incumbent's reserved gate
17260        /// and duplicate refusal behave exactly as before.
17261        #[test]
17262        fn without_an_open_swap_the_ordinary_gates_decide() {
17263            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17264            supervisor.close_swap("vault");
17265
17266            let candidate = handler
17267                .handle_control(
17268                    ConnectionId::new(2),
17269                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17270                )
17271                .unwrap();
17272            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17273            let duplicate = handler
17274                .handle_control(
17275                    ConnectionId::new(3),
17276                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17277                )
17278                .unwrap();
17279            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17280            assert!(registry.get_candidate("vault").unwrap().is_none());
17281        }
17282    }
17283
17284    /// `scope.sync` and `scope.describe` through the real control handler: who
17285    /// may sync is decided by the registration and launch nonce of the module
17286    /// connection, never by the request body.
17287    mod scopes {
17288        mod idle_tests {
17289            use super::*;
17290            include!("scope_idle_tests.rs");
17291        }
17292        mod route_open_timings {
17293            use super::*;
17294            include!("route_open_timing_tests.rs");
17295        }
17296        mod route_open_refusal_log {
17297            use super::*;
17298            include!("route_open_refusal_log_tests.rs");
17299        }
17300        mod activity {
17301            include!("activity_scope_tests.rs");
17302        }
17303        mod delta {
17304            include!("scope_handler_delta_tests.rs");
17305        }
17306        mod call_modes {
17307            include!("scope_call_mode_tests.rs");
17308        }
17309        #[tokio::test]
17310        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17311            let rig = rig().await;
17312            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17313                "s".into(),
17314                1,
17315                "tool".into(),
17316            ))
17317            .unwrap();
17318            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17319            let (direct, _rx) = wide_ctx(999);
17320            for (ctx, expected) in [
17321                (&rig.owner, FrameType::Response),
17322                (&direct, FrameType::Error),
17323            ] {
17324                let frame =
17325                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17326                        .unwrap();
17327                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17328                assert_eq!(replies[0].header.ty, expected);
17329                if expected == FrameType::Response {
17330                    let reply: subc_control::ScopeServedByReply =
17331                        serde_json::from_slice(&replies[0].body).unwrap();
17332                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17333                } else {
17334                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17335                }
17336            }
17337            let reply = call(
17338                &rig.handler,
17339                &rig.owner,
17340                &ModuleControlRequestFromModule::ScopeDescribe {
17341                    owner: Principal::Reserved {
17342                        module_id: OWNER.into(),
17343                    },
17344                    scope_ref: "s".into(),
17345                },
17346            )
17347            .await;
17348            assert!(matches!(
17349                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17350                ModuleControlResponseToModule::ScopeDescribe { .. }
17351            ));
17352        }
17353
17354        use subc_protocol::scope::{
17355            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17356            ScopeStatus,
17357        };
17358
17359        use super::*;
17360
17361        const OWNER: &str = "prefrontal-core";
17362
17363        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17364            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17365        }
17366
17367        async fn call(
17368            handler: &ControlHandler,
17369            ctx: &RouteCtx,
17370            request: &ModuleControlRequestFromModule,
17371        ) -> Frame {
17372            let body = serde_json::to_vec(request).unwrap();
17373            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17374            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17375            assert_eq!(replies.len(), 1, "{replies:?}");
17376            replies.pop().unwrap()
17377        }
17378
17379        async fn sync(
17380            handler: &ControlHandler,
17381            ctx: &RouteCtx,
17382            generation: u64,
17383            scopes: Vec<ScopeRecord>,
17384        ) -> Result<ModuleControlResponseToModule, String> {
17385            let reply = call(
17386                handler,
17387                ctx,
17388                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17389            )
17390            .await;
17391            match reply.header.ty {
17392                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17393                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17394            }
17395        }
17396
17397        async fn describe(
17398            handler: &ControlHandler,
17399            ctx: &RouteCtx,
17400            owner: &str,
17401            scope_ref: &str,
17402        ) -> ModuleControlResponseToModule {
17403            let reply = call(
17404                handler,
17405                ctx,
17406                &ModuleControlRequestFromModule::ScopeDescribe {
17407                    owner: Principal::Reserved {
17408                        module_id: owner.to_string(),
17409                    },
17410                    scope_ref: scope_ref.to_string(),
17411                },
17412            )
17413            .await;
17414            assert_eq!(
17415                reply.header.ty,
17416                FrameType::Response,
17417                "{:?}",
17418                parse_error(&reply)
17419            );
17420            serde_json::from_slice(&reply.body).unwrap()
17421        }
17422
17423        #[tokio::test]
17424        async fn scope_describe_never_echoes_bind_proof_or_hash() {
17425            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17426            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17427            let mut rig = rig().await;
17428            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17429                .await;
17430            let _bound = rig
17431                .bound(
17432                    Some(AFT),
17433                    PLEXUS,
17434                    Some(
17435                        rig.selector("s", Some(1))
17436                            .with_bind_proof(Some(PROOF.into())),
17437                    ),
17438                )
17439                .await;
17440            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17441            let output = serde_json::to_string(&reply).unwrap();
17442            assert!(output.contains("\"status\":\"live\""));
17443            assert!(!output.contains(PROOF));
17444            assert!(!output.contains(HASH));
17445        }
17446
17447        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
17448        async fn module(
17449            handler: &ControlHandler,
17450            connection: u64,
17451            module_id: &str,
17452            nonce: Option<&str>,
17453        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17454            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17455            hello_via_sink(
17456                handler,
17457                &ctx,
17458                &mut rx,
17459                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17460            )
17461            .await;
17462            (ctx, rx)
17463        }
17464
17465        /// `direct` and every other client connection has no registration, so
17466        /// it can neither sync nor own a scope.
17467        #[tokio::test]
17468        async fn a_client_connection_cannot_sync_or_describe() {
17469            let handler = ControlHandler::new(Arc::new(Registry::default()));
17470            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17471            for request in [
17472                ModuleControlRequestFromModule::ScopeSync {
17473                    generation: 1,
17474                    scopes: vec![head("s", 1)],
17475                },
17476                ModuleControlRequestFromModule::ScopeDescribe {
17477                    owner: Principal::Direct,
17478                    scope_ref: "s".to_string(),
17479                },
17480            ] {
17481                let reply = call(&handler, &ctx, &request).await;
17482                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17483            }
17484            assert!(
17485                !handler
17486                    .scopes
17487                    .read()
17488                    .unwrap()
17489                    .describe(
17490                        &Principal::Reserved {
17491                            module_id: OWNER.to_string()
17492                        },
17493                        "s"
17494                    )
17495                    .owner_synced
17496            );
17497        }
17498
17499        /// A module the supervisor did not spawn registers without a launch
17500        /// nonce, so it is never an owner's current launch.
17501        #[tokio::test]
17502        async fn a_module_without_a_supervised_launch_cannot_sync() {
17503            let handler = ControlHandler::new(Arc::new(Registry::default()));
17504            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17505            assert_eq!(
17506                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17507                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17508            );
17509        }
17510
17511        #[tokio::test]
17512        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17513            let supervisor = SupervisorHandle::new();
17514            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17515            let handler = ControlHandler::new(Arc::new(Registry::default()))
17516                .with_supervisor(supervisor.clone());
17517            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17518            sync(&handler, &incumbent, 1, vec![head("s", 1)])
17519                .await
17520                .expect("the current launch syncs");
17521
17522            // A swap candidate registers with the swap token and is refused
17523            // while the incumbent keeps syncing.
17524            supervisor.open_swap(OWNER, "n2".to_string());
17525            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17526            assert_eq!(
17527                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17528                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17529            );
17530            sync(&handler, &incumbent, 2, vec![head("s", 1)])
17531                .await
17532                .expect("the serving owner syncs during the swap");
17533
17534            // The swap fails and is rolled back. The candidate never held sync
17535            // authority, and still cannot sync.
17536            supervisor.close_swap(OWNER);
17537            assert_eq!(
17538                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17539                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17540            );
17541            sync(&handler, &incumbent, 3, vec![head("s", 1)])
17542                .await
17543                .expect("the serving owner syncs after the rollback");
17544            handler.cleanup_connection(candidate.connection_id).unwrap();
17545
17546            // A swap that cuts over. Promotion records the candidate's nonce as
17547            // the module's spawn nonce, which is what `set_spawn_nonce` does
17548            // here; the promoted connection then takes authority at any
17549            // generation and the superseded incumbent is refused.
17550            supervisor.open_swap(OWNER, "n3".to_string());
17551            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17552            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17553            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17554                .await
17555                .expect("the promoted launch takes authority");
17556            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17557                panic!("unexpected reply {reply:?}");
17558            };
17559            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17560            assert_eq!(
17561                sync(&handler, &incumbent, 4, Vec::new()).await,
17562                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17563            );
17564        }
17565
17566        /// Authority dies with its connection: the cleanup path releases it,
17567        /// so the owner's next connection takes it at any generation.
17568        #[tokio::test]
17569        async fn closing_the_authority_connection_frees_sync_authority() {
17570            let supervisor = SupervisorHandle::new();
17571            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17572            let handler = ControlHandler::new(Arc::new(Registry::default()))
17573                .with_supervisor(supervisor.clone());
17574            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17575            sync(&handler, &first, 10, vec![head("s", 1)])
17576                .await
17577                .unwrap();
17578            handler.cleanup_connection(first.connection_id).unwrap();
17579
17580            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17581            sync(&handler, &second, 1, vec![head("s", 1)])
17582                .await
17583                .expect("the next connection takes the released authority");
17584        }
17585
17586        #[tokio::test]
17587        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17588            let supervisor = SupervisorHandle::new();
17589            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17590            let handler =
17591                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17592            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17593            sync(&handler, &first, 10, vec![head("s", 1)])
17594                .await
17595                .unwrap();
17596            handler
17597                .handle_control_frame(
17598                    &first,
17599                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17600                )
17601                .await
17602                .unwrap();
17603            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17604            sync(&handler, &second, 1, vec![head("s", 1)])
17605                .await
17606                .expect("GOODBYE releases authority even if the old socket remains open");
17607        }
17608
17609        #[tokio::test]
17610        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17611            let registry = Arc::new(Registry::default());
17612            let supervisor_handle = SupervisorHandle::new();
17613            let supervisor =
17614                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17615                    .with_handle(supervisor_handle.clone())
17616                    .with_daemon_incarnation("incarnation-7".to_string());
17617            // Configured with enabled: false, so the supervisor lists the
17618            // module without spawning a process for it.
17619            supervisor
17620                .supervise_configured(
17621                    ModuleSpec {
17622                        module_id: OWNER.to_string(),
17623                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17624                        args: Vec::new(),
17625                        env: Vec::new(),
17626                        reserved: false,
17627                        reserved_prefixes: Vec::new(),
17628                        protocol: ModuleProtocol::Subc,
17629                        overlap: Default::default(),
17630                    },
17631                    false,
17632                )
17633                .unwrap();
17634            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17635            let handler =
17636                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17637            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17638
17639            // Configured but not yet synced: a reader waits for the owner.
17640            let ModuleControlResponseToModule::ScopeDescribe {
17641                status,
17642                daemon_incarnation,
17643                owner_synced,
17644                owner_configured,
17645                scope,
17646                ..
17647            } = describe(&handler, &reader, OWNER, "s").await
17648            else {
17649                panic!("not a describe reply");
17650            };
17651            assert_eq!(status, ScopeStatus::NotLive);
17652            assert_eq!(daemon_incarnation, "incarnation-7");
17653            assert!(!owner_synced);
17654            assert!(owner_configured);
17655            assert!(scope.is_none());
17656
17657            // Not a supervised module: the owner will never sync, and a reader
17658            // refuses rather than waits.
17659            let ModuleControlResponseToModule::ScopeDescribe {
17660                status,
17661                owner_configured,
17662                ..
17663            } = describe(&handler, &reader, "ghost", "s").await
17664            else {
17665                panic!("not a describe reply");
17666            };
17667            assert_eq!(status, ScopeStatus::NotLive);
17668            assert!(!owner_configured);
17669
17670            // Live, with the stamp fields and the computed owner_authorized.
17671            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17672            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17673            let ModuleControlResponseToModule::ScopeDescribe {
17674                status,
17675                scope_epoch,
17676                owner_synced,
17677                scope,
17678                ..
17679            } = describe(&handler, &reader, OWNER, "s").await
17680            else {
17681                panic!("not a describe reply");
17682            };
17683            assert_eq!(status, ScopeStatus::Live);
17684            assert_eq!(scope_epoch, Some(4));
17685            assert!(owner_synced);
17686            let stamp = scope.expect("a live scope carries its stamp");
17687            assert!(
17688                stamp.owner_authorized,
17689                "prefrontal-core is the default authority"
17690            );
17691            assert_eq!(stamp.kind, ScopeKind::Head);
17692        }
17693
17694        #[tokio::test]
17695        async fn scope_authority_owners_decides_owner_authorized() {
17696            let supervisor = SupervisorHandle::new();
17697            supervisor.set_spawn_nonce("broca", "b1".to_string());
17698            let handler = ControlHandler::new(Arc::new(Registry::default()))
17699                .with_supervisor(supervisor)
17700                .with_scope_authority_owners(vec!["broca".to_string()]);
17701            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17702            let mut gated = head("s", 1);
17703            gated.attributes.agent_id = Some("agent".to_string());
17704            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17705            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17706                describe(&handler, &broca, "broca", "s").await
17707            else {
17708                panic!("not a describe reply");
17709            };
17710            assert!(scope.unwrap().owner_authorized);
17711        }
17712
17713        /// With route admission, the stamp, the commit re-check and drains in
17714        /// place, the feature is advertised: the module ops in HELLO_ACK, and
17715        /// `scopes/v1` in HELLO_ACK and `server.describe`.
17716        #[tokio::test]
17717        async fn scope_ops_and_the_scopes_capability_are_advertised() {
17718            let handler = ControlHandler::new(Arc::new(Registry::default()));
17719            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17720            let ack = hello_via_sink(
17721                &handler,
17722                &ctx,
17723                &mut rx,
17724                hello_frame("m", PROTOCOL_VERSION, 1),
17725            )
17726            .await;
17727            let ack = parse_ack(&ack);
17728            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17729                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17730            }
17731            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17732
17733            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17734            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17735            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17736            let reply = handler
17737                .handle_control_frame(&client, frame)
17738                .await
17739                .unwrap()
17740                .pop()
17741                .unwrap();
17742            let ClientControlResponse::ServerDescribe { capabilities, .. } =
17743                serde_json::from_slice(&reply.body).unwrap()
17744            else {
17745                panic!("not a server.describe reply");
17746            };
17747            assert!(
17748                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17749                "{capabilities:?}"
17750            );
17751        }
17752
17753        // ---- route admission, stamps, commit re-check and drains ----------
17754
17755        const PLEXUS: &str = "plexus";
17756        const OTHER: &str = "other";
17757        const AFT: &str = "aft";
17758        const BROCA: &str = "broca";
17759        const MAGIC: &str = "magic-context";
17760
17761        fn nonce(module_id: &str) -> String {
17762            format!("nonce-{module_id}")
17763        }
17764
17765        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17766            let (tx, rx) = mpsc::channel(64);
17767            (
17768                RouteCtx {
17769                    connection_id: ConnectionId::new(connection),
17770                    egress: FrameSink::new(tx),
17771                },
17772                rx,
17773            )
17774        }
17775
17776        /// A daemon with a configured owner (prefrontal-core) registered on its
17777        /// own module connection, two routable targets (plexus, other), and
17778        /// launch nonces minted for the modules that open routes as carriers.
17779        struct Rig {
17780            handler: ControlHandler,
17781            forwarding: Arc<ForwardingTable>,
17782            owner: RouteCtx,
17783            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17784            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17785            generation: u64,
17786            next_connection: u64,
17787            _supervisor: Supervisor,
17788        }
17789
17790        async fn rig() -> Rig {
17791            rig_with_flow_support(true).await
17792        }
17793
17794        async fn rig_with_flow_support(flow_support: bool) -> Rig {
17795            rig_with_scope_capabilities(flow_support, false).await
17796        }
17797
17798        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17799            let mut provides = Vec::new();
17800            if flow_support {
17801                provides.push("flow-scopes/v1");
17802            }
17803            if run_support {
17804                provides.push("agent-run-scopes/v1");
17805            }
17806            rig_with_target_capabilities(&provides).await
17807        }
17808
17809        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17810            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17811            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17812            // Decoder versions alone cannot admit scoped calls: only a capability
17813            // declaration promises the required run or flow behaviour.
17814            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17815            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17816            Frame::build(
17817                FrameType::Hello,
17818                control_flags(),
17819                0,
17820                0,
17821                connection,
17822                serde_json::to_vec(&body).unwrap(),
17823            )
17824            .unwrap()
17825        }
17826
17827        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17828            let registry = Arc::new(Registry::default());
17829            let forwarding = Arc::new(ForwardingTable::default());
17830            let supervisor_handle = SupervisorHandle::new();
17831            let supervisor =
17832                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
17833                    .with_handle(supervisor_handle.clone());
17834            supervisor
17835                .supervise_configured(
17836                    ModuleSpec {
17837                        module_id: OWNER.to_string(),
17838                        program: PathBuf::from("/nonexistent/prefrontal-core"),
17839                        args: Vec::new(),
17840                        env: Vec::new(),
17841                        reserved: false,
17842                        reserved_prefixes: Vec::new(),
17843                        protocol: ModuleProtocol::Subc,
17844                        overlap: Default::default(),
17845                    },
17846                    false,
17847                )
17848                .unwrap();
17849            for module_id in [OWNER, AFT, BROCA, MAGIC] {
17850                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17851            }
17852            let handler =
17853                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17854                    .with_supervisor(supervisor_handle);
17855            let (owner, mut owner_rx) = wide_ctx(1);
17856            hello_via_sink(
17857                &handler,
17858                &owner,
17859                &mut owner_rx,
17860                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17861            )
17862            .await;
17863            let mut modules = BTreeMap::new();
17864            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17865                let (ctx, mut rx) = wide_ctx(connection);
17866                let hello = hello_with_capabilities(module_id, connection, provides);
17867                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17868                modules.insert(module_id.to_string(), (ctx, rx));
17869            }
17870            Rig {
17871                handler,
17872                forwarding,
17873                owner,
17874                _owner_rx: owner_rx,
17875                modules,
17876                generation: 0,
17877                next_connection: 100,
17878                _supervisor: supervisor,
17879            }
17880        }
17881
17882        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17883            ScopeCarrier::new(Principal::Reserved {
17884                module_id: module_id.to_string(),
17885            })
17886            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17887        }
17888
17889        /// The scope most tests open under: aft carries to any module, broca
17890        /// only to plexus and other, and the owner delegates as agent-1.
17891        fn session(scope_epoch: u64) -> ScopeRecord {
17892            let mut record = head("s", scope_epoch);
17893            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17894            record.attributes.agent_id = Some("agent-1".to_string());
17895            record.attributes.delegates = true;
17896            record
17897        }
17898
17899        impl Rig {
17900            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17901                self.generation += 1;
17902                sync(&self.handler, &self.owner, self.generation, scopes)
17903                    .await
17904                    .expect("the owner's sync is accepted");
17905            }
17906
17907            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17908                ScopeSelector::new(
17909                    Principal::Reserved {
17910                        module_id: OWNER.to_string(),
17911                    },
17912                    scope_ref.to_string(),
17913                    scope_epoch,
17914                )
17915            }
17916
17917            fn open_frame(
17918                &mut self,
17919                opener: Option<&str>,
17920                target: &str,
17921                scope: Option<ScopeSelector>,
17922            ) -> (
17923                RouteCtx,
17924                mpsc::Receiver<crate::router::OutboundFrame>,
17925                Frame,
17926            ) {
17927                self.next_connection += 1;
17928                let (ctx, rx) = wide_ctx(self.next_connection);
17929                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17930                    busy_refusals: None,
17931                    target: RouteTarget::ToolProvider {
17932                        module_id: target.to_string(),
17933                    },
17934                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
17935                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
17936                        module_id: module_id.to_string(),
17937                        launch_nonce: nonce(module_id),
17938                    }),
17939                    consumer_capabilities: None,
17940                    role_versions: None,
17941                    admission_facts: None,
17942                    scope,
17943                })
17944                .unwrap();
17945                let frame = Frame::build(
17946                    FrameType::Request,
17947                    control_flags(),
17948                    0,
17949                    0,
17950                    self.next_connection,
17951                    body,
17952                )
17953                .unwrap();
17954                (ctx, rx, frame)
17955            }
17956
17957            /// Open and expect a refusal before anything is relayed.
17958            async fn refused(
17959                &mut self,
17960                opener: Option<&str>,
17961                target: &str,
17962                scope: Option<ScopeSelector>,
17963            ) -> String {
17964                self.refusal_body(opener, target, scope).await["code"]
17965                    .as_str()
17966                    .unwrap()
17967                    .to_string()
17968            }
17969
17970            async fn refusal_body(
17971                &mut self,
17972                opener: Option<&str>,
17973                target: &str,
17974                scope: Option<ScopeSelector>,
17975            ) -> Value {
17976                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17977                let replies = tokio::time::timeout(
17978                    Duration::from_secs(2),
17979                    self.handler.handle_control_frame(&ctx, frame),
17980                )
17981                .await
17982                .expect("the open must be refused before waiting for a bind ack")
17983                .unwrap();
17984                assert_eq!(replies.len(), 1, "{replies:?}");
17985                assert_eq!(replies[0].header.ty, FrameType::Error);
17986                let (_, module_rx) = self.modules.get_mut(target).unwrap();
17987                assert!(
17988                    module_rx.try_recv().is_err(),
17989                    "a refused open relays nothing"
17990                );
17991                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17992                parse_error(&replies[0])
17993            }
17994
17995            /// Start an open and return its task and the bind the target got.
17996            async fn relayed(
17997                &mut self,
17998                opener: Option<&str>,
17999                target: &str,
18000                scope: Option<ScopeSelector>,
18001            ) -> Relayed {
18002                self.relayed_with_legacy_root(opener, target, scope, None)
18003                    .await
18004            }
18005
18006            async fn relayed_with_legacy_root(
18007                &mut self,
18008                opener: Option<&str>,
18009                target: &str,
18010                scope: Option<ScopeSelector>,
18011                legacy_root: Option<&std::path::Path>,
18012            ) -> Relayed {
18013                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18014                if let Some(root) = legacy_root {
18015                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18016                    body["identity"]["project_root"] = serde_json::json!(root);
18017                    frame.body = serde_json::to_vec(&body).unwrap();
18018                }
18019                let handler = self.handler.clone();
18020                let task_ctx = ctx.clone();
18021                let task = tokio::spawn(async move {
18022                    handler
18023                        .handle_control_frame(&task_ctx, frame)
18024                        .await
18025                        .unwrap()
18026                });
18027                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18028                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18029                    .await
18030                    .expect("the target receives the relayed route.bind")
18031                    .unwrap()
18032                    .frame;
18033                Relayed {
18034                    target: target.to_string(),
18035                    client: ctx,
18036                    client_rx: rx,
18037                    task,
18038                    bind,
18039                }
18040            }
18041
18042            async fn ack(&self, relayed: &Relayed) {
18043                let (module, _) = &self.modules[&relayed.target];
18044                self.handler
18045                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18046                    .await
18047                    .unwrap();
18048            }
18049
18050            /// Open, ack and return the bound route.
18051            async fn bound(
18052                &mut self,
18053                opener: Option<&str>,
18054                target: &str,
18055                scope: Option<ScopeSelector>,
18056            ) -> Bound {
18057                let relayed = self.relayed(opener, target, scope).await;
18058                self.commit_relayed(relayed).await
18059            }
18060
18061            async fn bound_with_legacy_root(
18062                &mut self,
18063                opener: Option<&str>,
18064                target: &str,
18065                scope: Option<ScopeSelector>,
18066                root: &std::path::Path,
18067            ) -> Bound {
18068                let relayed = self
18069                    .relayed_with_legacy_root(opener, target, scope, Some(root))
18070                    .await;
18071                self.commit_relayed(relayed).await
18072            }
18073
18074            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18075                self.ack(&relayed).await;
18076                let Relayed {
18077                    target,
18078                    client,
18079                    mut client_rx,
18080                    task,
18081                    bind,
18082                } = relayed;
18083                assert!(
18084                    task.await.unwrap().is_empty(),
18085                    "the open is answered by commit"
18086                );
18087                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18088                Bound {
18089                    target,
18090                    client,
18091                    client_rx,
18092                    channel,
18093                    epoch,
18094                    bind,
18095                }
18096            }
18097
18098            fn live(&self, route: &Bound) -> bool {
18099                matches!(
18100                    self.forwarding
18101                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18102                        .unwrap(),
18103                    DataRoute::Client(DataRouteState::Bound(_))
18104                )
18105            }
18106        }
18107
18108        struct Relayed {
18109            target: String,
18110            client: RouteCtx,
18111            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18112            task: tokio::task::JoinHandle<Vec<Frame>>,
18113            bind: Frame,
18114        }
18115
18116        struct Bound {
18117            target: String,
18118            client: RouteCtx,
18119            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18120            channel: u16,
18121            epoch: u32,
18122            bind: Frame,
18123        }
18124
18125        impl Bound {
18126            /// The reason of the `route.closed` this client was sent, after
18127            /// checking it also got a GOODBYE on exactly this route.
18128            fn closed_reason(&mut self) -> RouteCloseReason {
18129                let mut reason = None;
18130                let mut goodbye = false;
18131                while let Ok(outbound) = self.client_rx.try_recv() {
18132                    let frame = outbound.frame;
18133                    match frame.header.ty {
18134                        FrameType::Goodbye => {
18135                            assert_eq!(
18136                                (frame.header.channel, frame.header.epoch),
18137                                (self.channel, self.epoch)
18138                            );
18139                            goodbye = true;
18140                        }
18141                        FrameType::Push => {
18142                            let ClientControlPush::RouteClosed {
18143                                reason: r,
18144                                module_id,
18145                                ..
18146                            } = serde_json::from_slice(&frame.body).unwrap()
18147                            else {
18148                                panic!("unexpected push");
18149                            };
18150                            assert_eq!(module_id, self.target);
18151                            reason = Some(r);
18152                        }
18153                        other => panic!("unexpected frame {other:?}"),
18154                    }
18155                }
18156                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18157                reason.expect("the client is told why the route closed")
18158            }
18159
18160            fn untouched(&mut self) -> bool {
18161                self.client_rx.try_recv().is_err()
18162            }
18163
18164            fn stamp(&self) -> Option<ScopeStamp> {
18165                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18166                    ModuleControlRequest::RouteBind { scope, .. } => scope,
18167                    other => panic!("expected a route.bind, got {other:?}"),
18168                }
18169            }
18170        }
18171
18172        #[tokio::test]
18173        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18174        ) {
18175            let mut rig = rig().await;
18176            let mut record = session(1);
18177            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18178            record.child_owners = vec![Principal::Reserved {
18179                module_id: MAGIC.to_string(),
18180            }];
18181            rig.sync(vec![record]).await;
18182            let scope = || Some(rig_selector("s", Some(1)));
18183
18184            // Admitted: the owner, a bare carrier to any module, a targeted
18185            // carrier to its listed module.
18186            rig.bound(Some(OWNER), PLEXUS, scope()).await;
18187            rig.bound(Some(AFT), OTHER, scope()).await;
18188            rig.bound(Some(BROCA), PLEXUS, scope()).await;
18189
18190            // Refused scope_not_carrier: a targeted carrier to an unlisted
18191            // module, a module that is not listed at all (a child owner is not
18192            // a carrier), and a direct key-holder.
18193            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18194                assert_eq!(
18195                    rig.refused(opener, target, scope()).await,
18196                    error_codes::SCOPE_NOT_CARRIER,
18197                    "{opener:?} -> {target}"
18198                );
18199            }
18200        }
18201
18202        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18203            ScopeSelector::new(
18204                Principal::Reserved {
18205                    module_id: OWNER.to_string(),
18206                },
18207                scope_ref.to_string(),
18208                scope_epoch,
18209            )
18210        }
18211
18212        #[tokio::test]
18213        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18214            let mut rig = rig().await;
18215            rig.sync(vec![session(1)]).await;
18216            for opener in [OWNER, AFT] {
18217                assert_eq!(
18218                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18219                        .await,
18220                    error_codes::SCOPE_EPOCH_REQUIRED,
18221                    "{opener}"
18222                );
18223            }
18224        }
18225
18226        #[tokio::test]
18227        async fn admission_separates_not_synced_not_live_and_ended() {
18228            let mut rig = rig().await;
18229            // Before the configured owner's first sync: retryable.
18230            let code = rig
18231                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18232                .await;
18233            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18234            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18235
18236            // An owner that is not configured will never sync: terminal.
18237            let ghost = ScopeSelector::new(
18238                Principal::Reserved {
18239                    module_id: "ghost".to_string(),
18240                },
18241                "s".to_string(),
18242                Some(1),
18243            );
18244            assert_eq!(
18245                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18246                error_codes::SCOPE_NOT_LIVE
18247            );
18248
18249            rig.sync(vec![session(2)]).await;
18250            assert_eq!(
18251                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18252                    .await,
18253                error_codes::SCOPE_NOT_LIVE
18254            );
18255            for epoch in [1, 3] {
18256                assert_eq!(
18257                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18258                        .await,
18259                    error_codes::SCOPE_ENDED,
18260                    "epoch {epoch}"
18261                );
18262            }
18263            // Control: the live epoch is admitted.
18264            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18265                .await;
18266        }
18267
18268        #[tokio::test]
18269        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18270            let mut rig = rig().await;
18271            rig.sync(vec![session(1)]).await;
18272            let route = rig
18273                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18274                .await;
18275            let stamp = route.stamp().expect("a scoped bind carries the stamp");
18276            assert_eq!(stamp.scope_ref, "s");
18277            assert_eq!(stamp.scope_epoch, 1);
18278            assert_eq!(stamp.kind, ScopeKind::Head);
18279            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18280            assert!(stamp.attributes.delegates);
18281            assert!(stamp.owner_authorized);
18282            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18283            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18284
18285            // broca owns a scope of its own on its own module connection; it is
18286            // not in scope_authority_owners, so its stamp is not authorized.
18287            let (broca, mut broca_rx) = wide_ctx(50);
18288            hello_via_sink(
18289                &rig.handler,
18290                &broca,
18291                &mut broca_rx,
18292                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18293            )
18294            .await;
18295            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18296                .await
18297                .unwrap();
18298            let own = ScopeSelector::new(
18299                Principal::Reserved {
18300                    module_id: BROCA.to_string(),
18301                },
18302                "b".to_string(),
18303                Some(1),
18304            );
18305            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18306            assert!(!route.stamp().unwrap().owner_authorized);
18307        }
18308
18309        #[tokio::test]
18310        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18311            let mut rig = rig().await;
18312            let mut record = head("s", 1);
18313            record.carriers = vec![carrier(AFT, None)];
18314            let flow_id = "Flow:run-7/step_2!~";
18315            record.attributes.flow_id = Some(flow_id.to_string());
18316            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18317                .await
18318                .unwrap();
18319            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18320                panic!("not a sync reply");
18321            };
18322            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18323            let route = rig
18324                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18325                .await;
18326            assert!(rig.live(&route), "the stamped bind committed");
18327            let stamp = route.stamp().expect("a flow scope carries a stamp");
18328            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18329            assert_eq!(stamp.attributes.agent_id, None);
18330            assert!(!stamp.attributes.delegates);
18331            assert!(stamp.owner_authorized);
18332            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18333            assert_eq!(unscoped.stamp(), None);
18334        }
18335
18336        #[tokio::test]
18337        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18338            let mut rig = rig_with_flow_support(false).await;
18339            let mut record = session(1);
18340            record.attributes.flow_id = Some("flow:7".to_string());
18341            rig.sync(vec![record]).await;
18342            for opener in [OWNER, AFT] {
18343                let body = rig
18344                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18345                    .await;
18346                assert_eq!(body["code"], "target_flow_unsupported");
18347                let message = body["message"].as_str().unwrap();
18348                for required in [PLEXUS, "flow-scopes/v1"] {
18349                    assert!(message.contains(required), "{message}");
18350                }
18351            }
18352        }
18353
18354        #[tokio::test]
18355        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18356            let mut rig = rig_with_flow_support(true).await;
18357            let mut record = session(1);
18358            record.attributes.flow_id = Some("flow:7".to_string());
18359            rig.sync(vec![record]).await;
18360            for opener in [OWNER, AFT] {
18361                let route = rig
18362                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18363                    .await;
18364                assert!(rig.live(&route));
18365                assert_eq!(
18366                    route.stamp().unwrap().attributes.flow_id.as_deref(),
18367                    Some("flow:7")
18368                );
18369            }
18370        }
18371
18372        #[tokio::test]
18373        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18374            use std::future::Future;
18375
18376            let mut rig = rig_with_flow_support(true).await;
18377            let mut record = session(1);
18378            record.attributes.flow_id = Some("flow:7".to_string());
18379            rig.sync(vec![record]).await;
18380            let (client, mut client_rx, frame) =
18381                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18382            // Hold the route-open response permit so admission sees the first
18383            // target but relay reservation cannot capture an endpoint yet.
18384            for _ in 0..64 {
18385                client.egress.try_send(route_bind_ack(1)).unwrap();
18386            }
18387            let handler = rig.handler.clone();
18388            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18389            std::future::poll_fn(|cx| {
18390                assert!(open.as_mut().poll(cx).is_pending());
18391                std::task::Poll::Ready(())
18392            })
18393            .await;
18394            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18395
18396            let old_connection = rig.modules[PLEXUS].0.connection_id;
18397            rig.handler.cleanup_connection(old_connection).unwrap();
18398            let (replacement, mut replacement_rx) = wide_ctx(200);
18399            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18400            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18401            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18402            let hello = Frame::build(
18403                FrameType::Hello,
18404                control_flags(),
18405                0,
18406                0,
18407                200,
18408                serde_json::to_vec(&body).unwrap(),
18409            )
18410            .unwrap();
18411            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18412
18413            client_rx.try_recv().unwrap();
18414            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18415                .await
18416                .expect("the replacement is refused without waiting for a bind ack")
18417                .unwrap();
18418            assert_eq!(replies.len(), 1);
18419            let body = parse_error(&replies[0]);
18420            assert_eq!(body["code"], "target_flow_unsupported");
18421            for required in [PLEXUS, "flow-scopes/v1"] {
18422                assert!(body["message"].as_str().unwrap().contains(required));
18423            }
18424            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18425            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18426            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18427            assert!(rig
18428                .handler
18429                .registry
18430                .get_module_by_connection(replacement.connection_id)
18431                .unwrap()
18432                .is_some());
18433        }
18434
18435        #[tokio::test]
18436        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18437            use std::future::Future;
18438            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18439            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18440            const NEXT_HASH: &str =
18441                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18442            let mut rig = rig().await;
18443            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18444            record.carriers.push(ScopeCarrier::new(Principal::Direct));
18445            rig.sync(vec![record.clone()]).await;
18446            let (client, mut rx, frame) = rig.open_frame(
18447                None,
18448                PLEXUS,
18449                Some(
18450                    rig.selector("s", Some(1))
18451                        .with_bind_proof(Some(PROOF.into())),
18452                ),
18453            );
18454            // Hold egress reservation while the scope owner changes its bind-proof
18455            // hash, so a proof valid at admission must be checked again at relay.
18456            for _ in 0..64 {
18457                client.egress.try_send(route_bind_ack(1)).unwrap();
18458            }
18459            let handler = rig.handler.clone();
18460            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18461            std::future::poll_fn(|cx| {
18462                assert!(open.as_mut().poll(cx).is_pending());
18463                std::task::Poll::Ready(())
18464            })
18465            .await;
18466            record.bind_proof_sha256 = Some(NEXT_HASH.into());
18467            rig.sync(vec![record]).await;
18468            rx.try_recv().unwrap();
18469            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18470                .await
18471                .unwrap()
18472                .unwrap();
18473            assert_eq!(
18474                parse_error(&replies[0])["code"],
18475                error_codes::SCOPE_BIND_PROOF_MISMATCH
18476            );
18477            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18478            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18479            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18480        }
18481
18482        #[tokio::test]
18483        async fn scope_bind_proof_reserved_openers_ignore_proof() {
18484            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18485            let mut rig = rig().await;
18486            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18487                .await;
18488            for opener in [OWNER, AFT] {
18489                for proof in [None, Some("malformed+proof=")] {
18490                    let route = rig
18491                        .bound(
18492                            Some(opener),
18493                            PLEXUS,
18494                            Some(
18495                                rig.selector("s", Some(1))
18496                                    .with_bind_proof(proof.map(str::to_string)),
18497                            ),
18498                        )
18499                        .await;
18500                    assert!(rig.live(&route));
18501                }
18502            }
18503        }
18504
18505        #[tokio::test]
18506        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18507            use std::future::Future;
18508            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18509            const NEXT_HASH: &str =
18510                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18511            let mut rig = rig().await;
18512            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18513                .await;
18514            let (client, mut rx, frame) =
18515                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18516            for _ in 0..64 {
18517                client.egress.try_send(route_bind_ack(1)).unwrap();
18518            }
18519            let handler = rig.handler.clone();
18520            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18521            std::future::poll_fn(|cx| {
18522                assert!(open.as_mut().poll(cx).is_pending());
18523                std::task::Poll::Ready(())
18524            })
18525            .await;
18526            rig.sync(vec![
18527                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18528            ])
18529            .await;
18530            rx.try_recv().unwrap();
18531            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18532            let bind = std::future::poll_fn(|cx| {
18533                assert!(open.as_mut().poll(cx).is_pending());
18534                module_rx.poll_recv(cx)
18535            })
18536            .await
18537            .unwrap()
18538            .frame;
18539            handler
18540                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18541                .await
18542                .unwrap();
18543            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18544                .await
18545                .unwrap()
18546                .unwrap();
18547            assert_eq!(replies.len(), 1);
18548            // Proof rechecking allowed the relay. Independent tag settlement
18549            // still asks the opener to retry after any same-epoch content change.
18550            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18551            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18552            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18553            let route = rig
18554                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18555                .await;
18556            assert!(rig.live(&route));
18557        }
18558
18559        #[tokio::test]
18560        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18561        {
18562            let mut rig = rig_with_flow_support(false).await;
18563            rig.sync(vec![session(1)]).await;
18564            let route = rig
18565                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18566                .await;
18567            assert!(rig.live(&route));
18568            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18569            let mut record = session(1);
18570            record.attributes.flow_id = Some("flow:7".to_string());
18571            rig.sync(vec![record]).await;
18572            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18573            assert!(rig.live(&unscoped));
18574            assert_eq!(unscoped.stamp(), None);
18575        }
18576
18577        #[tokio::test]
18578        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18579            let mut rig = rig().await;
18580            let mut record = session(1);
18581            record.attributes.flow_id = Some("flow:7".to_string());
18582            rig.sync(vec![record.clone()]).await;
18583            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18584            let mut owner_route = rig
18585                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18586                .await;
18587            let mut carrier_route = rig
18588                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18589                .await;
18590            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18591            rig.sync(vec![record.clone()]).await;
18592            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18593            assert!(rig.live(&owner_route) && owner_route.untouched());
18594            assert!(rig.live(&carrier_route) && carrier_route.untouched());
18595
18596            record.attributes.flow_id = Some("flow:8".to_string());
18597            rig.sync(vec![record]).await;
18598            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18599            let before = before.unwrap();
18600            assert_eq!(after.scope_epoch, before.scope_epoch);
18601            assert!(after.version > before.version);
18602            for route in [&mut owner_route, &mut carrier_route] {
18603                assert!(!rig.live(route));
18604                assert_eq!(
18605                    route.closed_reason(),
18606                    RouteCloseReason::ScopeDelegationChanged
18607                );
18608            }
18609            assert!(rig.live(&unscoped) && unscoped.untouched());
18610            // Each provider also receives a GOODBYE for its drained route;
18611            // consume it before expecting the next route.bind on that sink.
18612            for target in [PLEXUS, OTHER] {
18613                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18614                let goodbye = module_rx
18615                    .try_recv()
18616                    .expect("the provider sees the drain")
18617                    .frame;
18618                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18619                assert!(module_rx.try_recv().is_err());
18620            }
18621            let rebound = rig
18622                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18623                .await;
18624            assert_eq!(
18625                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18626                Some("flow:8")
18627            );
18628        }
18629
18630        /// The owner's sync lands between admission and the module's ack. The
18631        /// open is refused by name, the module's other routes stay up, and the
18632        /// reserved pair is released. Changed content is retryable; an ended
18633        /// scope is not.
18634        #[tokio::test]
18635        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18636            let mut rig = rig().await;
18637            rig.sync(vec![session(1)]).await;
18638            let mut cotenant = rig
18639                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18640                .await;
18641
18642            let mut changed = session(1);
18643            changed.child_owners.push(Principal::Reserved {
18644                module_id: MAGIC.to_string(),
18645            });
18646            let mut ended = None;
18647            for (code, next) in [
18648                (error_codes::SCOPE_CHANGED, vec![changed]),
18649                (error_codes::SCOPE_ENDED, Vec::new()),
18650            ] {
18651                let relayed = rig
18652                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18653                    .await;
18654                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18655                ended = Some(next.is_empty());
18656                rig.sync(next).await;
18657                rig.ack(&relayed).await;
18658                let replies = relayed.task.await.unwrap();
18659                assert_eq!(replies.len(), 1, "{replies:?}");
18660                assert_eq!(parse_error(&replies[0])["code"], code);
18661                assert_eq!(
18662                    subc_protocol::error_codes::is_retryable_route_open(code),
18663                    code == error_codes::SCOPE_CHANGED
18664                );
18665                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18666                // The module is told to drop just the binding it created.
18667                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18668                // Collected, because ending the scope also closes the co-tenant
18669                // route, whose GOODBYE comes first.
18670                let mut goodbyes = Vec::new();
18671                while let Ok(outbound) = plexus_rx.try_recv() {
18672                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18673                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18674                }
18675                assert!(
18676                    goodbyes.contains(&(bind_channel, bind_epoch)),
18677                    "{goodbyes:?}"
18678                );
18679                assert!(rig
18680                    .handler
18681                    .registry
18682                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18683                    .unwrap()
18684                    .is_some());
18685            }
18686            assert_eq!(ended, Some(true));
18687            // The co-tenant stayed up through the change, and closed only when
18688            // the scope ended, by the drain rule rather than by the commit.
18689            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18690        }
18691
18692        /// Each row of the drain table on one set of routes: the owner's, a
18693        /// bare carrier's, and a targeted carrier's to each of its targets.
18694        #[tokio::test]
18695        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18696            struct Case {
18697                name: &'static str,
18698                change: fn(&mut ScopeRecord),
18699                /// Closed routes by index: owner->plexus, aft->plexus,
18700                /// broca->plexus, broca->other.
18701                closed: [Option<RouteCloseReason>; 4],
18702            }
18703            use RouteCloseReason::*;
18704            let cases = [
18705                Case {
18706                    name: "a carrier entry removed",
18707                    change: |r| {
18708                        r.carriers.retain(|c| {
18709                            c.principal
18710                                != Principal::Reserved {
18711                                    module_id: AFT.to_string(),
18712                                }
18713                        })
18714                    },
18715                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18716                },
18717                Case {
18718                    name: "a target removed from a carrier",
18719                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18720                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
18721                },
18722                Case {
18723                    name: "a bare carrier narrowed to targets",
18724                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18725                    closed: [None, Some(ScopeCarrierRemoved), None, None],
18726                },
18727                Case {
18728                    name: "delegates turned off",
18729                    change: |r| r.attributes.delegates = false,
18730                    closed: [Some(ScopeDelegationChanged); 4],
18731                },
18732                Case {
18733                    name: "agent_id changed",
18734                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18735                    closed: [Some(ScopeDelegationChanged); 4],
18736                },
18737                Case {
18738                    name: "a carrier added, child owners changed, the record re-sent",
18739                    change: |r| {
18740                        r.carriers.push(carrier(MAGIC, None));
18741                        r.child_owners.push(Principal::Reserved {
18742                            module_id: MAGIC.to_string(),
18743                        });
18744                    },
18745                    closed: [None; 4],
18746                },
18747                Case {
18748                    name: "a target added",
18749                    change: |r| {
18750                        r.carriers[1]
18751                            .targets
18752                            .as_mut()
18753                            .unwrap()
18754                            .push("third".to_string())
18755                    },
18756                    closed: [None; 4],
18757                },
18758                Case {
18759                    name: "delegates turned on",
18760                    change: |r| r.attributes.delegates = true,
18761                    closed: [None; 4],
18762                },
18763            ];
18764            for case in cases {
18765                let mut rig = rig().await;
18766                rig.sync(vec![session(1)]).await;
18767                let scope = || Some(rig_selector("s", Some(1)));
18768                let mut routes = [
18769                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
18770                    rig.bound(Some(AFT), PLEXUS, scope()).await,
18771                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
18772                    rig.bound(Some(BROCA), OTHER, scope()).await,
18773                ];
18774                let mut record = session(1);
18775                (case.change)(&mut record);
18776                rig.sync(vec![record]).await;
18777                for (index, expected) in case.closed.iter().enumerate() {
18778                    let route = &mut routes[index];
18779                    match expected {
18780                        Some(reason) => {
18781                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
18782                            assert_eq!(
18783                                route.closed_reason(),
18784                                *reason,
18785                                "{}: route {index}",
18786                                case.name
18787                            );
18788                        }
18789                        None => {
18790                            assert!(rig.live(route), "{}: route {index} closed", case.name);
18791                            assert!(
18792                                route.untouched(),
18793                                "{}: route {index} was told something",
18794                                case.name
18795                            );
18796                        }
18797                    }
18798                }
18799            }
18800        }
18801
18802        #[tokio::test]
18803        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18804            // Removed, and replaced by a higher epoch.
18805            for next in [Vec::new(), vec![session(2)]] {
18806                let mut rig = rig().await;
18807                rig.sync(vec![session(1)]).await;
18808                let mut route = rig
18809                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18810                    .await;
18811                rig.sync(next).await;
18812                assert!(!rig.live(&route));
18813                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18814            }
18815
18816            // A child whose parent ends: its routes close as parent-ended, the
18817            // child stays live, and routes under the parent close as ended.
18818            let mut rig = rig().await;
18819            let mut child = session(1);
18820            child.scope_ref = "child".to_string();
18821            child.kind = ScopeKind::Worker;
18822            child.parent = Some(ScopeParent::new(
18823                Principal::Reserved {
18824                    module_id: OWNER.to_string(),
18825                },
18826                "s".to_string(),
18827                1,
18828            ));
18829            rig.sync(vec![session(1), child.clone()]).await;
18830            let mut child_route = rig
18831                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18832                .await;
18833            assert_eq!(
18834                child_route.stamp().unwrap().parent_state,
18835                Some(ParentState::Linked)
18836            );
18837            rig.sync(vec![child]).await;
18838            assert!(!rig.live(&child_route));
18839            assert_eq!(
18840                child_route.closed_reason(),
18841                RouteCloseReason::ScopeParentEnded
18842            );
18843        }
18844
18845        #[tokio::test]
18846        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18847        ) {
18848            let mut rig = rig().await;
18849            rig.sync(vec![session(1)]).await;
18850            let mut route = rig
18851                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18852                .await;
18853            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18854            rig.sync(vec![session(1)]).await;
18855            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18856            assert!(rig.live(&route) && route.untouched());
18857
18858            // A call in flight on the route when another carrier is added. A
18859            // forwarded REQUEST holds one credit on the route's flow until the
18860            // module answers; the router takes it exactly like this.
18861            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18862                .forwarding
18863                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18864                .unwrap()
18865            else {
18866                panic!("the route is bound");
18867            };
18868            binding.flow.acquire_tagged(9, false).await.unwrap();
18869            let mut widened = session(1);
18870            widened.carriers.push(carrier(MAGIC, None));
18871            rig.sync(vec![widened]).await;
18872            assert!(rig.live(&route) && route.untouched());
18873            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18874            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18875            // The call's credit is still held on an open flow, so its answer
18876            // will be delivered: closing the route would have closed the flow.
18877            assert_eq!(binding.flow.in_flight(), 1);
18878            binding
18879                .flow
18880                .acquire_tagged(10, false)
18881                .await
18882                .expect("the flow is still open");
18883        }
18884
18885        /// A swap's superseded endpoint keeps its routes until drained; ending
18886        /// the scope closes them there too.
18887        #[tokio::test]
18888        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18889            let mut rig = rig().await;
18890            rig.sync(vec![session(1)]).await;
18891            let mut on_incumbent = rig
18892                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18893                .await;
18894
18895            // Swap plexus: register a candidate and cut over, leaving the
18896            // incumbent superseded with the route still on it.
18897            let (candidate, _candidate_rx) = wide_ctx(9);
18898            let registration = rig
18899                .handler
18900                .registry
18901                .register_candidate_with_control_ops(
18902                    manifest(PLEXUS, PROTOCOL_VERSION),
18903                    PROTOCOL_VERSION,
18904                    candidate.connection_id,
18905                    module_baseline_control_ops(),
18906                )
18907                .unwrap();
18908            rig.forwarding
18909                .register_candidate_module_connection(
18910                    candidate.connection_id,
18911                    PLEXUS.to_string(),
18912                    PROTOCOL_VERSION,
18913                    manifest_concurrency(&registration.manifest),
18914                    candidate.egress.clone(),
18915                )
18916                .unwrap();
18917            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18918            rig.handler
18919                .registry
18920                .promote_candidate(PLEXUS)
18921                .unwrap()
18922                .unwrap();
18923            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18924
18925            rig.sync(Vec::new()).await;
18926            assert!(!rig.live(&on_incumbent));
18927            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18928            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18929            let goodbye = incumbent_rx
18930                .try_recv()
18931                .expect("the superseded endpoint is told")
18932                .frame;
18933            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18934        }
18935    }
18936}
18937
18938#[cfg(test)]
18939mod concurrency_default_exposure_tests {
18940    use super::*;
18941
18942    fn hello_body(role_json: &str) -> Vec<u8> {
18943        format!(
18944            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18945        )
18946        .into_bytes()
18947    }
18948
18949    fn manifest_from(body: &[u8]) -> ModuleManifest {
18950        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18951        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18952            .expect("manifest parses")
18953    }
18954
18955    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18956
18957    #[test]
18958    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18959        let body = hello_body(&format!(
18960            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18961        ));
18962        let manifest = manifest_from(&body);
18963        // Precondition: serde really resolved it to the default, so the typed
18964        // manifest alone cannot answer the question this probe exists for.
18965        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18966        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18967    }
18968
18969    #[test]
18970    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18971        let body = hello_body(&format!(
18972            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18973        ));
18974        let manifest = manifest_from(&body);
18975        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18976        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18977    }
18978
18979    #[test]
18980    fn non_management_roles_are_never_reported() {
18981        let body = hello_body(
18982            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18983        );
18984        let manifest = manifest_from(&body);
18985        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18986    }
18987}