Skip to main content

subc_daemon/
control.rs

1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6    fmt,
7    net::SocketAddr,
8    path::{Path, PathBuf},
9    sync::{Arc, Mutex, RwLock},
10    time::{Duration, Instant as StdInstant},
11};
12
13use serde::{Deserialize, Serialize};
14use subc_control::{
15    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
16    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
17    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
18    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
19    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
20    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
21    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
22};
23use subc_protocol::{
24    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
25    error_codes,
26    manifest::{
27        validate_hello_capability_grammar, validate_hello_confirm_operations,
28        validate_hello_display_name, validate_hello_event_declarations,
29        validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
30        validate_hello_role_declarations, validate_hello_self_signal_declarations,
31        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
32        ProviderRole,
33    },
34    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
35    scope::{
36        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
37        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
38        SCOPE_SYNC_OP,
39    },
40    session::{
41        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
42        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
43        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
44        ROLE_VERSIONS_FIELD,
45    },
46    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
47    Priority, RouteTarget, PROTOCOL_VERSION,
48};
49use tokio::time::{timeout_at, Instant};
50use tracing::{debug, info, instrument::WithSubscriber, warn};
51
52use crate::{
53    caller_identity::CallerIdentity,
54    capability_requirements::{
55        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
56        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
57        RequirementStatus, RuntimeModule,
58    },
59    child_roster::DaemonShutdownFlag,
60    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
61    forwarding::{
62        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
63        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
64        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
65    },
66    observability::{
67        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
68    },
69    privileged_prompt::{prompt_text, PrivilegedOperation},
70    provenance::{
71        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
72    },
73    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
74    router::{daemon_error_flags, RouteCtx, RouterError},
75    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
76    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
77    stderr_tail::{CaptureState, TailEntry},
78    supervise::{
79        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
80        SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
81    },
82    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
83};
84
85/// Lowest envelope version this subc build will negotiate.
86///
87/// Module HELLO negotiation is exact: peers must use the daemon's locked
88/// protocol version. Older and newer peers receive `version_unsupported` and
89/// are not registered.
90pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
91
92const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
93const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
94const CAP_PING_PONG: &str = "ping_pong_v1";
95const CAP_SESSION_ATTACH: &str = "session_attach_v1";
96const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
97
98const SUBC_CONTROL_OPS: &[&str] = &[
99    ops::DAEMON_SHUTDOWN_INTENT,
100    ops::ACTIVITY_RECORD,
101    ops::ACTIVITY_QUERY,
102    ops::ACTIVITY_SUBSCRIBE,
103    ops::SERVER_DESCRIBE,
104    ops::CATALOG_LIST,
105    ops::ROUTE_OPEN,
106    ops::ROUTE_POLL,
107    ops::ROUTE_CLOSING,
108    ops::ROUTE_CLOSED,
109    ops::SUPERVISOR_LIST,
110    ops::SUPERVISOR_RESTART,
111    ops::SUPERVISOR_SWAP,
112    ops::SUPERVISOR_RELOAD,
113    ops::SUPERVISOR_RESCAN,
114    ops::SUPERVISOR_RELEASE_RESERVED,
115    ops::SUPERVISOR_SET_ENABLED,
116    ops::SUPERVISOR_HEALTH_PROBE,
117    ops::SUPERVISOR_HEALTH,
118    ops::SUPERVISOR_STDERR_TAIL,
119    ops::SUPERVISOR_TERMINALS,
120    ops::SUPERVISOR_ROUTES,
121    ops::SUPERVISOR_PROVENANCE,
122    ops::SUPERVISOR_SPAWN_SNAPSHOT,
123    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
124];
125
126/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
127/// announced by a capability in `server.describe` instead, and clients check
128/// that capability before sending it. `ck daemon --json` prints `subc_ops`
129/// verbatim and scripts read that output, so a new name there would change it.
130/// A malformed body for one of these is still refused `invalid_control_body`
131/// rather than as an unknown op.
132const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
133
134mod agent_commands;
135mod rescan_checked;
136use rescan_checked::{RescanMode, RescanPlan};
137
138const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
139    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
140    "supervisor.live_roots",
141    "supervisor.agent_commands",
142    SCOPE_SYNC_OP,
143    SCOPE_APPLY_OP,
144    SCOPE_DESCRIBE_OP,
145    "operator.confirm",
146];
147
148/// Module-originated ops the daemon answers but does not advertise in
149/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
150/// belongs to is incomplete, so no module is told it works before it does.
151const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
152
153const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
154
155/// How long subc waits for a module to ack a relayed route.bind before returning
156/// `module_timeout`. The ack waits on the module's own configure, which for AFT
157/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
158/// and DB-open work — on a cold page cache or a large repo that legitimately
159/// exceeds a couple of seconds. The default is generous because rejecting a VALID
160/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
161/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
162pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
163
164/// How many CONSECUTIVE full-budget relay timeouts against one target module
165/// open that module's bind-relay breaker.
166///
167/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
168/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
169/// so three consecutive full-budget timeouts take ~36s to observe: every client
170/// whose open contributed to opening the breaker had already given up on its
171/// own. That is what makes opening the breaker unable to turn a call that would
172/// have succeeded into a refusal — it can only make an already-failing module
173/// fail faster.
174///
175/// Two would be reachable inside one default deadline. One would convict a
176/// module on a single cold-cache bind, which is exactly the valid-but-slow case
177/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
178pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
179
180/// How long a module's bind-relay breaker stays open before exactly one
181/// `route.open` is let through as a probe.
182///
183/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
184/// re-pays a full-budget stall almost continuously, and the breaker stops being
185/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
186/// deadline: a client that starts retrying after the module recovers has to get
187/// a probe opportunity inside its own deadline, or the breaker converts a
188/// recovered module into a failed call — the failure it exists to prevent,
189/// pointed the other way.
190///
191/// 20s sits between those with room on both sides, and it caps what a wedged
192/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
193/// rather than one per `route.open` per connection. The stall that motivated
194/// this, with its measurements, is written up in
195/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
196/// waited the whole budget out.
197pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
198
199const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
200const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
201
202fn reload_verdict(
203    configured: &Path,
204    spawned_from: Option<&Path>,
205    image: subc_control::RunningImageAgreement,
206) -> PendingReloadVerdict {
207    let path = match spawned_from {
208        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
209            ReloadPathAgreement::Match
210        }
211        Some(spawned_from) => ReloadPathAgreement::Mismatch {
212            configured: configured.to_path_buf(),
213            spawned_from: spawned_from.to_path_buf(),
214        },
215        None => ReloadPathAgreement::Unavailable {
216            reason: if matches!(
217                image,
218                subc_control::RunningImageAgreement::Unavailable {
219                    reason: subc_control::RunningImageUnavailableReason::NotRunning
220                }
221            ) {
222                ReloadPathUnavailableReason::NotRunning
223            } else {
224                ReloadPathUnavailableReason::SpawnedPathUnavailable
225            },
226        },
227    };
228    PendingReloadVerdict::new(path, image)
229}
230
231fn launch_spec_agreement(
232    configured: &crate::supervise::ModuleSpec,
233    running: bool,
234    spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237    if !running {
238        return Agreement::Unavailable {
239            reason: Reason::NotRunning,
240        };
241    }
242    match spawned {
243        None => Agreement::Unavailable {
244            reason: Reason::SpawnedSpecUnavailable,
245        },
246        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247            Agreement::Match
248        }
249        Some(_) => Agreement::Mismatch,
250    }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254    if configured == spawned_from {
255        return true;
256    }
257    #[cfg(windows)]
258    {
259        // Windows spawn records the resolved verbatim path (\\?\...), while
260        // configuration keeps its original spelling. Resolve both with the OS
261        // instead of stripping prefixes or folding case, which could conflate
262        // distinct names in case-sensitive directories. Different resolved
263        // paths still differ even if their files have identical content.
264        match (
265            std::fs::canonicalize(configured),
266            std::fs::canonicalize(spawned_from),
267        ) {
268            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269            _ => false,
270        }
271    }
272    #[cfg(not(windows))]
273    {
274        false
275    }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280    build: DaemonBuildProvenance,
281    pid: Option<u32>,
282    started_at_ms: Option<u64>,
283    start_clock: Option<crate::clock::StartClock>,
284    executable_path: Option<PathBuf>,
285    executable_identity: Option<SpawnedFileIdentity>,
286    process_start_time: Option<u64>,
287    probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291    fn default() -> Self {
292        Self {
293            build: DaemonBuildProvenance {
294                build_git_sha: None,
295                build_lock_digest: None,
296            },
297            pid: None,
298            started_at_ms: None,
299            start_clock: None,
300            executable_path: None,
301            executable_identity: None,
302            process_start_time: None,
303            probe: ExecutableIdentityProbe::default(),
304        }
305    }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310    supervisor: Supervisor,
311    config_path: PathBuf,
312    configured_port: Option<u16>,
313    storage_config: Option<crate::daemon_config::StorageConfig>,
314    admission_facts_carrier_module_id: Option<String>,
315    admission_facts_targets: Option<Vec<String>>,
316    scope_authority_owners: Vec<String>,
317    federated_scope_owner: String,
318    privileged_operator_prompt: PrivilegedOperatorPrompt,
319    /// Counts executed rescans that set out to change the supervised modules'
320    /// staged or applied configuration, counted before the first change so a
321    /// rescan that fails partway still counts. With the daemon incarnation it
322    /// names the state a `supervisor.rescan_checked` preview was computed
323    /// against.
324    generation: Arc<std::sync::atomic::AtomicU64>,
325}
326
327/// Refusal labels passed to `observe_route_open_refusal` that mean the target
328/// module is not serving right now, and so open or extend an outage in the
329/// route outage tracker. Every one of them is only reachable after the target
330/// was found in the registry, which is what keeps an arbitrary client-chosen
331/// id from ever creating tracker state.
332///
333/// Deliberately absent: `not_registered` and `removed` (the id may be
334/// anything a client sent, and a removed module is gone on purpose),
335/// `protocol_none` (such a module never serves routes, so nothing is out),
336/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
337/// capability and admission-facts refusals (they refuse the caller, not a
338/// module outage), and `relay_reservation_failed` (its code ranges over
339/// capacity limits as well as a vanished connection). Capacity, breaker,
340/// relay-timeout and module-rejection refusals do not pass through that
341/// function at all; the breaker logs its own transitions.
342///
343/// The two not-serving refusals that bypass that function record themselves
344/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
345/// `required_capability_unprovided` is not tracked: the module itself is up,
346/// and the outage belongs to the missing provider.
347const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
348    "reloading",
349    "supervisor_not_live",
350    "registration_not_active",
351    "no_forwarding_connection",
352    "relay_send_failed",
353];
354
355/// Real channel-0 control handler for subc itself.
356#[derive(Clone)]
357pub struct ControlHandler {
358    registry: Arc<Registry>,
359    activity: Option<crate::activity::ActivityLedger>,
360    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
361    forwarding: Arc<ForwardingTable>,
362    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
363    supervisor: SupervisorHandle,
364    pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
365    agent_declarations: agent_commands::Queues,
366    list_admission: Arc<tokio::sync::Semaphore>,
367    subc_capabilities: Arc<[String]>,
368    /// Daemon-wide route.bind relay budget. Used as the fallback when the
369    /// target module has no per-module override in
370    /// `route_bind_relay_timeouts`.
371    route_bind_relay_timeout: Duration,
372    /// Per-module route.bind relay budget overrides, keyed by module id. When
373    /// `handle_route_open` resolves the deadline for a target module, a
374    /// per-module entry wins over the daemon-wide value above.
375    route_bind_relay_timeouts: BTreeMap<String, Duration>,
376    /// Per-target-module bind-relay breaker state. Shared with the forwarding
377    /// table, which is where a new module connection resets it.
378    route_bind_breakers: RouteBindBreakers,
379    /// Live relay admissions keyed by target module. Shared through the
380    /// forwarding table so cloned or separately built handlers enforce one cap.
381    route_bind_concurrency: RouteBindConcurrency,
382    /// Start and end of each module's not-serving period as seen by
383    /// `route.open`, so an outage gets one line at each edge instead of only
384    /// the per-refusal INFO lines. Taken from the forwarding table, so every
385    /// handler built over one table shares it.
386    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
387    /// Consecutive relay timeouts that open a module's breaker.
388    route_bind_breaker_threshold: u32,
389    /// How long a breaker stays open before one probe is admitted.
390    route_bind_breaker_cooldown: Duration,
391    health_probe_timeout: Duration,
392    /// Central storage policy. When set, each registering module receives its
393    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
394    storage_config: Option<crate::daemon_config::StorageConfig>,
395    /// The machine id established at boot, served on every HELLO_ACK and on
396    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
397    /// changes the file, never this value. `None` serves no id.
398    machine_id: Option<crate::machine_id::MachineId>,
399    admission_facts_carrier_module_id: Option<String>,
400    admission_facts_targets: Option<Vec<String>>,
401    /// Scope records with their sync authorities and tombstones; see
402    /// `crate::scopes`. Shared by clones of this handler, so every connection
403    /// reads and writes one table.
404    scopes: Arc<RwLock<ScopeTable>>,
405    /// Wakes the expiry loop when an accepted scope change may have added a
406    /// deadline, so the loop can wait without ticking while no live scope has
407    /// one. A permit is stored when nothing waits, so no wake is lost.
408    scope_deadline_added: Arc<tokio::sync::Notify>,
409    /// The configured `scope_authority_owners`, kept so a rescan can report a
410    /// changed value as needing a daemon restart; rescan never applies it.
411    scope_authority_owners: Vec<String>,
412    federated_scope_owner: String,
413    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
414    privileged_operator_prompt: PrivilegedOperatorPrompt,
415    /// The local and peer address of each accepted TCP client connection, read
416    /// from the daemon's own socket, so a privileged request can name its caller.
417    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
418    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
419    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
420    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
421    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
422    pub(crate) operator_apps:
423        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
424    /// One launch entry per app, behind a leaf lock: no other lock is taken
425    /// while it is held. Entries are memory-only, so a daemon or
426    /// installed-service restart drops them and a relaunched daemon never
427    /// mistakes an older process for its own launch.
428    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
429    /// Serializes `app.open` launches: the liveness check in the decision and
430    /// the later publish must not interleave across concurrent calls, or two
431    /// calls could both launch and the second publish would orphan the first
432    /// process. Refusals and `already_running` answers share the same lock;
433    /// `app.open` is rare, so the cost is negligible.
434    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
435    // Portable wire samples exercise dispatch without starting an OS process.
436    #[cfg(test)]
437    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
438    #[cfg(test)]
439    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
440    /// Test-only Windows signature and flash hooks. Production builds carry
441    /// neither: with the publisher unset, the production table never reaches
442    /// verification, and the default attention hook only logs.
443    #[allow(dead_code)]
444    #[cfg(windows)]
445    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
446    #[allow(dead_code)]
447    #[cfg(windows)]
448    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
449    /// Test-only tracking fault hook: makes opening the tracking handle on
450    /// the launched pid fail, so a test can prove an untrackable launch is
451    /// stopped rather than left running.
452    #[allow(dead_code)]
453    #[cfg(windows)]
454    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
455    /// Test-only count of Windows spawn attempts, so a test can prove a
456    /// refused launch started no process. Unit tests only.
457    #[cfg(all(test, windows))]
458    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
459    #[cfg(test)]
460    caller_lookup: Option<TestCallerLookup>,
461    /// The launch nonce each module connection presented at HELLO, which is how
462    /// a `scope.sync` is matched to the owner's current launch.
463    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
464    rescan: Option<SupervisorRescanContext>,
465    connected_clients: ConnectedClients,
466    counters: DaemonCounters,
467    /// Why this daemon serves no local endpoint, when binding one failed. `None`
468    /// when the endpoint is served, and also on a handler that was never told.
469    local_endpoint_unavailable: Option<String>,
470    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
471    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
472    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
473    /// their usual severity (see [`log_requirement_events`]). A handler built
474    /// without one has a flag that is never set.
475    daemon_shutdown: DaemonShutdownFlag,
476    daemon_provenance: DaemonProvenanceFacts,
477    #[cfg(test)]
478    control_dispatch_delay: Option<Duration>,
479    #[cfg(test)]
480    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
481    /// Holds `route.open` after the route.bind is queued to the module and
482    /// before it is marked enqueued, so a test can make the module's reply (or
483    /// a client teardown) land exactly inside that window.
484    #[cfg(test)]
485    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
486    /// Holds a HELLO after its registry entry is reserved and before its
487    /// forwarding endpoint is installed, so a test can read the catalog and
488    /// open routes while the module is registered but not yet routable. A
489    /// blocking barrier because the HELLO handler is synchronous; the test
490    /// runs the HELLO on its own thread.
491    #[cfg(test)]
492    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
493}
494
495impl fmt::Debug for ControlHandler {
496    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
497        f.debug_struct("ControlHandler")
498            .field("registry", &self.registry)
499            .field("forwarding", &self.forwarding)
500            .field("process_liveness", &self.process_liveness.is_some())
501            .field("supervisor", &self.supervisor)
502            .field("subc_capabilities", &self.subc_capabilities)
503            .finish()
504    }
505}
506
507struct RouteOpenRequest {
508    target: RouteTarget,
509    identity: BindIdentity,
510    consumer_identity: Option<ConsumerIdentity>,
511    consumer_capabilities: Option<Vec<String>>,
512    role_versions: Option<BTreeMap<String, String>>,
513    busy_refusals: Option<bool>,
514    admission_facts: Option<serde_json::Value>,
515    scope: Option<ScopeSelector>,
516}
517
518#[derive(Clone, Copy)]
519enum RouteOpenPhase {
520    Admission,
521    ProjectRoot,
522    EgressReservation,
523    BindSend,
524    BindAckWait,
525}
526
527/// Store elapsed time for each route-open phase, including attempts that end
528/// in an early refusal or backend error. Admission also includes rechecks and
529/// commit bookkeeping performed between the other timed phases.
530struct RouteOpenTimings {
531    module_id: String,
532    principal: String,
533    admission: Duration,
534    project_root: Duration,
535    egress_reservation: Duration,
536    bind_send: Duration,
537    bind_ack_wait: Duration,
538    phase: Option<RouteOpenPhase>,
539    phase_started_at: Instant,
540}
541
542impl RouteOpenTimings {
543    fn new() -> Self {
544        Self {
545            module_id: String::new(),
546            // Refusals before attestation must not claim a verified principal.
547            principal: "unresolved".to_string(),
548            admission: Duration::ZERO,
549            project_root: Duration::ZERO,
550            egress_reservation: Duration::ZERO,
551            bind_send: Duration::ZERO,
552            bind_ack_wait: Duration::ZERO,
553            phase: Some(RouteOpenPhase::Admission),
554            phase_started_at: Instant::now(),
555        }
556    }
557
558    fn finish(&mut self) {
559        let elapsed = self.phase_started_at.elapsed();
560        match self.phase.take() {
561            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
562            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
563            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
564            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
565            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
566            None => {}
567        }
568    }
569
570    fn enter(&mut self, phase: RouteOpenPhase) {
571        self.finish();
572        self.phase = Some(phase);
573        self.phase_started_at = Instant::now();
574    }
575}
576
577struct RouteBindReservationGuard {
578    forwarding: Arc<ForwardingTable>,
579    endpoint: ModuleEndpointId,
580    relay_corr: u64,
581    armed: bool,
582}
583
584struct ModuleControlRpcGuard {
585    forwarding: Arc<ForwardingTable>,
586    endpoint: ModuleEndpointId,
587    corr: u64,
588    armed: bool,
589}
590
591impl ModuleControlRpcGuard {
592    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
593        Self {
594            forwarding,
595            endpoint,
596            corr,
597            armed: true,
598        }
599    }
600
601    fn disarm(&mut self) {
602        self.armed = false;
603    }
604}
605
606impl Drop for ModuleControlRpcGuard {
607    fn drop(&mut self) {
608        if self.armed {
609            let _ = self
610                .forwarding
611                .cancel_module_control_rpc(self.endpoint, self.corr);
612        }
613    }
614}
615
616impl RouteBindReservationGuard {
617    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
618        Self {
619            forwarding,
620            endpoint,
621            relay_corr,
622            armed: true,
623        }
624    }
625
626    fn release_and_disarm(&mut self) {
627        if !self.armed {
628            return;
629        }
630        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
631            self.endpoint,
632            self.relay_corr,
633            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
634        ) {
635            send_goodbye_target_best_effort(
636                &self.forwarding.counters(),
637                &target,
638                "canceled route.bind",
639            );
640        }
641        self.armed = false;
642    }
643
644    fn disarm(&mut self) {
645        self.armed = false;
646    }
647}
648
649impl Drop for RouteBindReservationGuard {
650    fn drop(&mut self) {
651        self.release_and_disarm();
652    }
653}
654
655/// Per-target-module circuit breaker around the `route.bind` relay.
656///
657/// The connection reader is serial per connection, so a module whose `on_bind`
658/// sits on the ack blocks every LATER frame on the connections that call it,
659/// including calls to unrelated modules. This does not make any module's bind
660/// fast; it stops the daemon paying the full budget again and again for a
661/// condition it has already observed.
662///
663/// State is keyed by TARGET MODULE and shared by every connection: a wedged
664/// module wedges everyone, so what one connection learned should protect the
665/// rest.
666///
667/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
668/// relay to that module has actually timed out, and is removed again when a
669/// relay is accepted or the module reconnects, so it cannot grow with traffic
670/// or with modules that behave.
671///
672/// # Why a `std` mutex here is not the head-of-line defect again
673///
674/// Acquisition never awaits. The critical section is a hash lookup plus a few
675/// integer updates, with no I/O and no `.await` inside it, so a reader task
676/// cannot be descheduled behind it the way it can behind
677/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
678/// primitive, held for the same kind of work, as the refusal counter this very
679/// path already increments.
680///
681/// It is also NOT on the data-plane splice path: only `route.open` and module
682/// registration touch it, so bound-route frames gain no state check and no
683/// contention.
684#[derive(Debug, Clone, Default)]
685pub(crate) struct RouteBindBreakers {
686    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
687}
688
689#[derive(Debug, Clone, Default)]
690pub(crate) struct RouteBindConcurrency {
691    modules: Arc<Mutex<HashMap<String, usize>>>,
692}
693
694struct RouteBindConcurrencyGuard {
695    concurrency: RouteBindConcurrency,
696    module_id: String,
697}
698
699impl RouteBindConcurrency {
700    /// Admit without waiting. Waiting here would move the bind stall from the
701    /// module reply to a semaphore and restore reader head-of-line blocking.
702    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
703        let mut modules = self
704            .modules
705            .lock()
706            .expect("route.bind concurrency mutex poisoned");
707        let in_flight = modules.entry(module_id.to_string()).or_default();
708        if *in_flight >= limit {
709            return Err(*in_flight);
710        }
711        *in_flight += 1;
712        Ok(RouteBindConcurrencyGuard {
713            concurrency: self.clone(),
714            module_id: module_id.to_string(),
715        })
716    }
717}
718
719impl Drop for RouteBindConcurrencyGuard {
720    fn drop(&mut self) {
721        let mut modules = self
722            .concurrency
723            .modules
724            .lock()
725            .expect("route.bind concurrency mutex poisoned");
726        let remove = {
727            let in_flight = modules
728                .get_mut(&self.module_id)
729                .expect("admitted route.bind has a concurrency entry");
730            *in_flight -= 1;
731            *in_flight == 0
732        };
733        if remove {
734            modules.remove(&self.module_id);
735        }
736    }
737}
738
739#[derive(Debug, Default)]
740struct ModuleBreakerState {
741    /// Relay timeouts observed with no accepted relay in between.
742    consecutive_timeouts: u32,
743    /// `Some` while the breaker is open: the instant the cooldown expires and
744    /// the next arrival may probe. `None` means closed.
745    cooldown_until: Option<Instant>,
746    /// A half-open probe has been admitted and has not settled yet. This is
747    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
748    /// that read the cooldown, so concurrent opens arriving at the moment the
749    /// cooldown expires cannot all decide that they are the probe.
750    probe_in_flight: Option<Arc<()>>,
751}
752
753/// What the breaker decided for one `route.open`, before any relay work.
754enum RouteBindAdmission<'a> {
755    Admitted {
756        guard: RouteBindBreakerGuard<'a>,
757        /// This open is the single half-open probe, so the transition is worth
758        /// one log line.
759        probe: bool,
760    },
761    Refused {
762        consecutive_timeouts: u32,
763        /// What is left of the cooldown. Zero when the refusal is because the
764        /// one probe is already in flight rather than because the cooldown has
765        /// not elapsed.
766        retry_in: Duration,
767        probe_in_flight: bool,
768    },
769}
770
771/// An outstanding admission, which must be told how its relay settled.
772///
773/// `Drop` settles it as inconclusive, so an early return between admission and
774/// the relay -- or the whole handler being cancelled when the client
775/// disconnects -- releases a half-open probe slot instead of leaving the
776/// breaker wedged half-open with no further probes.
777struct RouteBindBreakerGuard<'a> {
778    breakers: RouteBindBreakers,
779    module_id: &'a str,
780    probe_token: Option<Arc<()>>,
781    settled: bool,
782}
783
784impl RouteBindBreakerGuard<'_> {
785    /// The module answered within the budget and took the bind. THE ONLY
786    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
787    /// breaker, which is a transition worth logging.
788    fn record_accepted(&mut self) -> bool {
789        self.settled = true;
790        self.breakers.record_accepted(self.module_id)
791    }
792
793    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
794    /// COUNTS TOWARD OPENING.
795    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
796        self.settled = true;
797        self.breakers.record_timeout(
798            self.module_id,
799            self.probe_token.as_ref(),
800            threshold,
801            cooldown,
802        )
803    }
804
805    /// Everything else: the module REJECTED the bind, its connection went away
806    /// mid-relay, or the waiter was cancelled.
807    ///
808    /// None of these is evidence that a module is slow, and each already has
809    /// its own refusal with its own code. A module that rejects a bind in
810    /// microseconds is healthy and must never be convicted for it; a module
811    /// that died has said nothing about the module that replaces it. So these
812    /// neither increment nor reset the count -- they only release a probe slot.
813    fn record_inconclusive(&mut self) {
814        self.settled = true;
815        self.breakers
816            .record_inconclusive(self.module_id, self.probe_token.as_ref());
817    }
818}
819
820impl Drop for RouteBindBreakerGuard<'_> {
821    fn drop(&mut self) {
822        if !self.settled {
823            self.breakers
824                .record_inconclusive(self.module_id, self.probe_token.as_ref());
825        }
826    }
827}
828
829/// The breaker moved to open, reported so the caller can log it outside the
830/// lock. Opening is rare and load-bearing; the refusals that follow are
831/// frequent and are counted rather than logged.
832struct BreakerOpened {
833    consecutive_timeouts: u32,
834    /// True when a failed probe re-opened an already-open breaker, which reads
835    /// very differently in a log from a first opening.
836    reopened_after_probe: bool,
837}
838
839impl RouteBindBreakers {
840    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
841        self.modules
842            .lock()
843            .expect("route.bind breaker mutex poisoned")
844    }
845
846    /// Decide whether this `route.open` may attempt its relay. Takes the map
847    /// lock and nothing else, and never awaits.
848    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
849        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
850            probe: probe_token.is_some(),
851            guard: RouteBindBreakerGuard {
852                breakers: self.clone(),
853                module_id,
854                probe_token,
855                settled: false,
856            },
857        };
858
859        let mut modules = self.lock();
860        let Some(state) = modules.get_mut(module_id) else {
861            return admitted(None);
862        };
863        let Some(cooldown_until) = state.cooldown_until else {
864            return admitted(None);
865        };
866        if state.probe_in_flight.is_some() {
867            return RouteBindAdmission::Refused {
868                consecutive_timeouts: state.consecutive_timeouts,
869                retry_in: Duration::ZERO,
870                probe_in_flight: true,
871            };
872        }
873        let now = Instant::now();
874        if now < cooldown_until {
875            return RouteBindAdmission::Refused {
876                consecutive_timeouts: state.consecutive_timeouts,
877                retry_in: cooldown_until - now,
878                probe_in_flight: false,
879            };
880        }
881        let token = Arc::new(());
882        state.probe_in_flight = Some(Arc::clone(&token));
883        admitted(Some(token))
884    }
885
886    fn record_accepted(&self, module_id: &str) -> bool {
887        self.lock()
888            .remove(module_id)
889            .is_some_and(|state| state.cooldown_until.is_some())
890    }
891
892    fn record_timeout(
893        &self,
894        module_id: &str,
895        probe_token: Option<&Arc<()>>,
896        threshold: u32,
897        cooldown: Duration,
898    ) -> Option<BreakerOpened> {
899        let mut modules = self.lock();
900        let state = modules.entry(module_id.to_string()).or_default();
901        let was_open = state.cooldown_until.is_some();
902        let was_probe = Self::owns_probe(state, probe_token);
903        if was_probe {
904            state.probe_in_flight = None;
905        }
906        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
907        if state.consecutive_timeouts < threshold {
908            return None;
909        }
910        state.cooldown_until = Some(Instant::now() + cooldown);
911        Some(BreakerOpened {
912            consecutive_timeouts: state.consecutive_timeouts,
913            reopened_after_probe: was_open && was_probe,
914        })
915    }
916
917    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
918        // A relay can finish after the breaker was reset or after it opened
919        // again and started a new probe. Only the guard whose token matches the
920        // active probe may release it, so a late relay never frees a newer probe.
921        state
922            .probe_in_flight
923            .as_ref()
924            .zip(token)
925            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
926    }
927
928    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
929        if let Some(state) = self.lock().get_mut(module_id) {
930            if Self::owns_probe(state, probe_token) {
931                state.probe_in_flight = None;
932            }
933        }
934    }
935
936    /// Discard what was learned about a module, because the process it was
937    /// learned about is gone. Returns the discarded count when it was non-zero.
938    ///
939    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
940    /// `module_id` is a configuration identity that outlives any particular
941    /// child; what the breaker observed was the process behind the module
942    /// connection of the moment. When a new connection registers under that id
943    /// the verdict's subject no longer exists, so the verdict is stale by
944    /// construction rather than merely likely to be wrong. Keeping it would
945    /// apply a dead process's record to a live one, which is the same defect
946    /// class this breaker exists to stop the daemon committing.
947    ///
948    /// A half-open probe in flight is discarded with the rest: it was a
949    /// question about the old process.
950    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
951        self.lock()
952            .remove(module_id)
953            .map(|state| state.consecutive_timeouts)
954            .filter(|discarded| *discarded > 0)
955    }
956
957    /// Open breakers, for the `server.describe` counters object. `None` when
958    /// none is open, so the key stays absent rather than present-and-empty.
959    ///
960    /// This is the operator's answer to "is this module refusing instantly or
961    /// is it fine?", which look identical from a client that retries and then
962    /// succeeds.
963    fn open_snapshot(&self) -> Option<serde_json::Value> {
964        let now = Instant::now();
965        let modules = self.lock();
966        let open = modules
967            .iter()
968            .filter_map(|(module_id, state)| {
969                let cooldown_until = state.cooldown_until?;
970                Some((
971                    module_id.clone(),
972                    serde_json::json!({
973                        "consecutive_timeouts": state.consecutive_timeouts,
974                        "cooldown_remaining_ms":
975                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
976                        "probe_in_flight": state.probe_in_flight.is_some(),
977                    }),
978                ))
979            })
980            .collect::<serde_json::Map<String, serde_json::Value>>();
981        (!open.is_empty()).then_some(serde_json::Value::Object(open))
982    }
983}
984
985impl ControlHandler {
986    pub fn new(registry: Arc<Registry>) -> Self {
987        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
988    }
989
990    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
991        let counters = forwarding.counters();
992        // Taken from the forwarding table rather than created here, so that the
993        // breaker a `route.open` consults is the same one a module's
994        // registration resets, however many handlers are built over one table.
995        let route_bind_breakers = forwarding.route_bind_breakers();
996        let route_bind_concurrency = forwarding.route_bind_concurrency();
997        let route_outages = forwarding.route_outages();
998        Self {
999            registry,
1000            activity: None,
1001            activity_streams: Arc::new(Mutex::new(HashMap::new())),
1002            forwarding,
1003            process_liveness: None,
1004            supervisor: SupervisorHandle::new(),
1005            resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1006                env!("CARGO_PKG_VERSION").to_owned(),
1007            )),
1008            agent_declarations: agent_commands::Queues::default(),
1009            list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1010            subc_capabilities: Arc::from([
1011                CAP_MANIFEST_REGISTRATION.to_string(),
1012                CAP_CHANNEL_LIFECYCLE.to_string(),
1013                CAP_PING_PONG.to_string(),
1014                CAP_SESSION_ATTACH.to_string(),
1015                CAP_ADMISSION_FACTS_RELAY.to_string(),
1016                CAP_SCOPES_V1.to_string(),
1017                subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1018                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1019                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1020                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1021                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1022                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1023                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1024                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1025                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1026                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1027                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1028                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1029            ]),
1030            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1031            route_bind_relay_timeouts: BTreeMap::new(),
1032            route_bind_breakers,
1033            route_bind_concurrency,
1034            route_outages,
1035            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1036            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1037            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1038            storage_config: None,
1039            machine_id: None,
1040            admission_facts_carrier_module_id: None,
1041            admission_facts_targets: None,
1042            scopes: Arc::new(RwLock::new(ScopeTable::new(
1043                crate::daemon_config::default_scope_authority_owners(),
1044            ))),
1045            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1046            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1047            federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1048            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1049            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1050            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1051            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1052            local_connections: Arc::new(Mutex::new(HashSet::new())),
1053            operator_app_policy: Default::default(),
1054            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1055            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1056            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1057            #[cfg(test)]
1058            app_open_test_platform: None,
1059            #[cfg(test)]
1060            app_open_test_launch: None,
1061            #[cfg(windows)]
1062            windows_verify: None,
1063            #[cfg(windows)]
1064            windows_flash: None,
1065            #[cfg(windows)]
1066            windows_track: None,
1067            #[cfg(all(test, windows))]
1068            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1069            #[cfg(test)]
1070            caller_lookup: None,
1071            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1072            rescan: None,
1073            connected_clients: ConnectedClients::new(),
1074            local_endpoint_unavailable: None,
1075            counters,
1076            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1077            daemon_shutdown: DaemonShutdownFlag::default(),
1078            daemon_provenance: DaemonProvenanceFacts::default(),
1079            #[cfg(test)]
1080            control_dispatch_delay: None,
1081            #[cfg(test)]
1082            provenance_probe_override: None,
1083            #[cfg(test)]
1084            route_bind_relay_sent_hold: None,
1085            #[cfg(test)]
1086            hello_registry_to_forwarding_hold: None,
1087        }
1088    }
1089
1090    /// Set the central storage policy: registering modules then receive their
1091    /// resolved storage descriptor in HELLO_ACK.
1092    pub fn with_storage_config(
1093        mut self,
1094        storage_config: Option<crate::daemon_config::StorageConfig>,
1095    ) -> Self {
1096        self.storage_config = storage_config;
1097        self
1098    }
1099
1100    /// Open the ledger on its own writer thread; an unavailable sink does not
1101    /// prevent the daemon from serving routing or supervisor requests.
1102    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1103        self.activity = Some(crate::activity::ActivityLedger::open(
1104            run_dir.join("activity.jsonl"),
1105            incarnation,
1106        ));
1107        self.supervisor.set_activity(self.activity.clone());
1108        self
1109    }
1110
1111    /// Set the machine id served to every registering module (HELLO_ACK) and on
1112    /// `server.describe`.
1113    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1114        self.machine_id = machine_id;
1115        self
1116    }
1117
1118    /// Configure the exact reserved module and target ids permitted to relay
1119    /// opaque admission facts. Config-file loading validates this authority;
1120    /// this builder keeps the same policy available to embedded test daemons.
1121    pub fn with_admission_facts_config(
1122        mut self,
1123        carrier_module_id: Option<String>,
1124        targets: Option<Vec<String>>,
1125    ) -> Self {
1126        self.admission_facts_carrier_module_id = carrier_module_id;
1127        self.admission_facts_targets = targets;
1128        self
1129    }
1130
1131    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1132    /// Replaces the scope table with an empty one under the new list, so call it
1133    /// while building the handler, before any module can sync.
1134    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1135        self.scopes = Arc::new(RwLock::new(
1136            ScopeTable::new(owners.iter().cloned())
1137                .with_federated_owner(self.federated_scope_owner.clone()),
1138        ));
1139        self.scope_authority_owners = owners;
1140        self
1141    }
1142
1143    /// Set the exclusive federated owner while building the handler, before syncs.
1144    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1145        self.scopes = Arc::new(RwLock::new(
1146            ScopeTable::new(self.scope_authority_owners.iter().cloned())
1147                .with_federated_owner(owner.clone()),
1148        ));
1149        self.federated_scope_owner = owner;
1150        self
1151    }
1152
1153    /// Set the startup policy before installing the rescan context or serving requests.
1154    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1155        self.privileged_operator_prompt = mode;
1156        self
1157    }
1158
1159    /// Override the route.bind relay timeout. Used by tests that assert the
1160    /// timeout path so they don't block on the production-safe default.
1161    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1162        self.route_bind_relay_timeout = timeout;
1163        self
1164    }
1165
1166    /// Install per-module route.bind relay budget overrides. A module id
1167    /// listed here wins over the daemon-wide default set via
1168    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1169    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1170    /// the same `Duration` the bind path will use.
1171    pub fn with_route_bind_relay_timeouts(
1172        mut self,
1173        timeouts: impl IntoIterator<Item = (String, Duration)>,
1174    ) -> Self {
1175        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1176        self
1177    }
1178
1179    /// Resolve the route.bind relay budget for a specific target module id.
1180    /// Per-module overrides win; the daemon-wide value (set via
1181    /// `with_route_bind_relay_timeout` or the built-in default) is the
1182    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1183    /// the same resolution `handle_route_open` will use.
1184    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1185        self.route_bind_relay_timeouts
1186            .get(module_id)
1187            .copied()
1188            .unwrap_or(self.route_bind_relay_timeout)
1189    }
1190
1191    /// Override the per-module bind-relay breaker policy.
1192    ///
1193    /// Used by tests, which cannot spend three production budgets opening a
1194    /// breaker or twenty seconds waiting for its cooldown. The production
1195    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1196    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1197    /// reasoning for the numbers.
1198    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1199        self.route_bind_breaker_threshold = threshold.max(1);
1200        self.route_bind_breaker_cooldown = cooldown;
1201        self
1202    }
1203
1204    #[cfg(test)]
1205    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1206        self.health_probe_timeout = timeout;
1207        self
1208    }
1209
1210    #[cfg(test)]
1211    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1212        self.control_dispatch_delay = Some(delay);
1213        self
1214    }
1215
1216    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1217    /// announce it is parked, once to be released. Build the barrier for two
1218    /// parties, the test being the other one.
1219    #[cfg(test)]
1220    pub(crate) fn with_route_bind_relay_sent_hold(
1221        mut self,
1222        hold: Arc<tokio::sync::Barrier>,
1223    ) -> Self {
1224        self.route_bind_relay_sent_hold = Some(hold);
1225        self
1226    }
1227
1228    /// A HELLO waits on `hold` twice between reserving its registry entry and
1229    /// installing its forwarding endpoint: once to announce it is parked, once
1230    /// to be released. Build the barrier for two parties, the test being the
1231    /// other one, and run the HELLO on a thread of its own.
1232    #[cfg(test)]
1233    pub(crate) fn with_hello_registry_to_forwarding_hold(
1234        mut self,
1235        hold: Arc<std::sync::Barrier>,
1236    ) -> Self {
1237        self.hello_registry_to_forwarding_hold = Some(hold);
1238        self
1239    }
1240
1241    #[cfg(test)]
1242    fn hold_between_registry_and_forwarding(&self) {
1243        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1244            hold.wait();
1245            hold.wait();
1246        }
1247    }
1248
1249    pub fn with_process_liveness(
1250        mut self,
1251        process_liveness: Arc<dyn ModuleProcessLiveness>,
1252    ) -> Self {
1253        self.process_liveness = Some(process_liveness);
1254        self
1255    }
1256
1257    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1258        self.supervisor = supervisor;
1259        self.supervisor.set_activity(self.activity.clone());
1260        self
1261    }
1262
1263    pub fn with_daemon_provenance(
1264        mut self,
1265        pid: u32,
1266        started_at_ms: u64,
1267        executable_path: Option<PathBuf>,
1268        build_git_sha: Option<String>,
1269        build_lock_digest: Option<String>,
1270    ) -> Self {
1271        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1272        let process_start_time = process_start_time(pid);
1273        self.daemon_provenance = DaemonProvenanceFacts {
1274            build: DaemonBuildProvenance {
1275                build_git_sha,
1276                build_lock_digest,
1277            },
1278            pid: Some(pid),
1279            started_at_ms: Some(started_at_ms),
1280            start_clock: None,
1281            executable_path,
1282            executable_identity,
1283            process_start_time,
1284            probe: ExecutableIdentityProbe::default(),
1285        };
1286        self
1287    }
1288
1289    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1290        self.daemon_provenance.start_clock = Some(clock);
1291        self
1292    }
1293
1294    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1295    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1296        self.scopes
1297            .write()
1298            .unwrap_or_else(|poisoned| poisoned.into_inner())
1299            .set_wall_clock(Arc::new(clock));
1300        self
1301    }
1302
1303    #[cfg(test)]
1304    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1305        self.provenance_probe_override = Some(result);
1306        self
1307    }
1308
1309    /// Install the configured module set and its reserved capability bindings.
1310    /// Bindings are configuration-scoped and may point at a provider that has not
1311    /// been installed yet, so this does not require the bound module to exist.
1312    pub fn with_capability_config(
1313        self,
1314        modules: impl IntoIterator<Item = (String, bool)>,
1315        reserved_capabilities: BTreeMap<String, String>,
1316    ) -> Self {
1317        self.capability_evaluator
1318            .configure(modules, reserved_capabilities);
1319        self
1320    }
1321
1322    /// Share the supervisor's daemon-shutdown flag with this handler, so
1323    /// requirement changes after the daemon has begun its announced shutdown
1324    /// are not reported as faults.
1325    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1326        self.daemon_shutdown = flag;
1327        self
1328    }
1329
1330    pub fn with_supervisor_rescan(
1331        mut self,
1332        supervisor: Supervisor,
1333        config_path: impl Into<PathBuf>,
1334        configured_port: Option<u16>,
1335    ) -> Self {
1336        self.rescan = Some(SupervisorRescanContext {
1337            supervisor,
1338            config_path: config_path.into(),
1339            configured_port,
1340            storage_config: self.storage_config.clone(),
1341            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1342            admission_facts_targets: self.admission_facts_targets.clone(),
1343            scope_authority_owners: self.scope_authority_owners.clone(),
1344            federated_scope_owner: self.federated_scope_owner.clone(),
1345            privileged_operator_prompt: self.privileged_operator_prompt,
1346            generation: Arc::default(),
1347        });
1348        self
1349    }
1350
1351    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1352        self.connected_clients = connected_clients;
1353        self
1354    }
1355
1356    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1357    /// an operator can see a bind failure without reading the daemon log.
1358    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1359        self.local_endpoint_unavailable = reason;
1360        self
1361    }
1362
1363    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1364        Arc::clone(&self.forwarding)
1365    }
1366
1367    pub(crate) fn counters(&self) -> DaemonCounters {
1368        self.counters.clone()
1369    }
1370
1371    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1372    /// requirement event without depending on an operator polling a status command.
1373    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1374        tokio::spawn(async move {
1375            loop {
1376                self.capability_evaluator
1377                    .wait_for_change_or_deadline()
1378                    .await;
1379                self.refresh_capability_requirements();
1380            }
1381        });
1382    }
1383
1384    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1385    /// deadline passed during sleep is enforced on the first tick after wake.
1386    /// While no live scope carries a deadline the loop waits for an accepted
1387    /// scope change instead of ticking, so a daemon without deadlines never
1388    /// wakes for expiry.
1389    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1390        tokio::spawn(async move {
1391            loop {
1392                let has_deadlines = self
1393                    .scopes
1394                    .read()
1395                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1396                    .has_deadlines();
1397                if !has_deadlines {
1398                    self.scope_deadline_added.notified().await;
1399                    continue;
1400                }
1401                tokio::time::sleep(Duration::from_secs(1)).await;
1402                if let Err(error) = self.sweep_expired_scopes() {
1403                    warn!(%error, "scope expiry sweep failed");
1404                }
1405            }
1406        });
1407    }
1408
1409    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1410        let mut table = self
1411            .scopes
1412            .write()
1413            .unwrap_or_else(|poisoned| poisoned.into_inner());
1414        let swept = table.sweep_expired();
1415        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1416        drop(table);
1417        self.close_scope_drained_routes(drained);
1418        Ok(())
1419    }
1420
1421    fn publish_scope_changes(
1422        &self,
1423        changes: &[crate::scopes::ScopeTagChange],
1424        expired: &[crate::scopes::ScopeExpired],
1425    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1426        if changes.is_empty() {
1427            return Ok(Vec::new());
1428        }
1429        let drained = self
1430            .forwarding
1431            .publish_scope_changes(changes)
1432            .map_err(RouterError::Forwarding)?;
1433        let mut counts = HashMap::new();
1434        for route in &drained {
1435            *counts
1436                .entry((
1437                    route.scope.owner.as_str(),
1438                    route.scope.scope_ref.as_str(),
1439                    route.scope.tag.scope_epoch,
1440                ))
1441                .or_insert(0usize) += 1;
1442        }
1443        for scope in expired {
1444            let routes_closed = counts
1445                .get(&(
1446                    scope.owner.as_str(),
1447                    scope.scope_ref.as_str(),
1448                    scope.scope_epoch,
1449                ))
1450                .copied()
1451                .unwrap_or(0);
1452            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1453                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1454                routes_closed, "scope expired");
1455        }
1456        Ok(drained)
1457    }
1458
1459    fn runtime_capability_snapshot(
1460        &self,
1461    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1462        let runtime = self
1463            .supervisor
1464            .list()
1465            .into_iter()
1466            .map(|module| {
1467                let status = module.status().map_err(|err| {
1468                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1469                })?;
1470                Ok(RuntimeModule {
1471                    module_id: status.module_id,
1472                    state: status.state,
1473                    enabled: status.enabled,
1474                })
1475            })
1476            .collect::<Result<Vec<_>, RouterError>>()?;
1477        // Include every registration, published or not. The capability
1478        // evaluator counts a module as providing its capabilities as soon as it
1479        // has registered, even before `catalog.list` shows it, so a HELLO in
1480        // progress does not change other modules' required-capability
1481        // verdicts. See `Registry::list_registered_modules`.
1482        let registrations = self.registry.list_registered_modules().map_err(|err| {
1483            RouterError::backend(
1484                0,
1485                0,
1486                format!("failed to list capability registrations: {err}"),
1487            )
1488        })?;
1489        let registrations = registrations
1490            .into_iter()
1491            .map(|registration| RegisteredModule {
1492                module_id: registration.manifest.module_id,
1493                module_version: registration.manifest.module_version,
1494                capabilities: registration.manifest.capabilities,
1495            })
1496            .collect();
1497        Ok((runtime, registrations))
1498    }
1499
1500    /// The capability side effects of a module becoming the active registration
1501    /// for its id: cache its manifest (logging how it differs from the cached
1502    /// one), run the deny census when its declarations call for one, and recompute the
1503    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1504    /// candidate's does not, and the supervisor does it at promotion instead,
1505    /// through [`crate::supervise::SwapPromotionObserver`].
1506    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1507        let cached_registration = RegisteredModule {
1508            module_id: registration.manifest.module_id.clone(),
1509            module_version: registration.manifest.module_version.clone(),
1510            capabilities: registration.manifest.capabilities.clone(),
1511        };
1512        let change = self.capability_evaluator.record_hello(&cached_registration);
1513        log_manifest_change(&cached_registration.module_id, &change);
1514        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1515            self.enforce_capability_denies();
1516        }
1517        self.refresh_capability_requirements();
1518    }
1519
1520    /// Point the shared supervisor handle at this handler for swap promotions.
1521    /// Called wherever a handler is put behind the `Arc` the router serves, so
1522    /// it can be held weakly.
1523    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1524        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1525            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1526        self.supervisor.set_swap_promotion_observer(observer);
1527    }
1528
1529    pub fn refresh_capability_requirements(&self) {
1530        match self.runtime_capability_snapshot() {
1531            Ok((runtime, registrations)) => {
1532                log_requirement_events(
1533                    self.capability_evaluator
1534                        .evaluate_now(&runtime, &registrations),
1535                    &self.daemon_shutdown,
1536                );
1537            }
1538            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1539        }
1540    }
1541
1542    /// Reconcile only live, attested route bindings after a capability deny edge
1543    /// or target claim was added. This is deliberately a control-plane census:
1544    /// the opaque forwarding hot path must not grow a per-frame capability check.
1545    fn enforce_capability_denies(&self) {
1546        let registrations = match self.registry.list_registered_modules() {
1547            Ok(snapshot) => snapshot,
1548            Err(err) => {
1549                warn!(error = %err, "failed to read registrations for capability deny census");
1550                return;
1551            }
1552        };
1553        let manifests = registrations
1554            .into_iter()
1555            .map(|registration| {
1556                (
1557                    registration.manifest.module_id.clone(),
1558                    registration.manifest,
1559                )
1560            })
1561            .collect::<BTreeMap<_, _>>();
1562        let census = match self.forwarding.route_census(None) {
1563            Ok(census) => census,
1564            Err(err) => {
1565                warn!(error = %err, "failed to read route census for capability deny enforcement");
1566                return;
1567            }
1568        };
1569
1570        for (target_module_id, routes) in census {
1571            let Some(target_manifest) = manifests.get(&target_module_id) else {
1572                continue;
1573            };
1574            let mut closed_routes = Vec::new();
1575            let mut goodbyes = Vec::new();
1576            for route in routes {
1577                let Principal::Reserved {
1578                    module_id: opening_module_id,
1579                } = &route.principal
1580                else {
1581                    continue;
1582                };
1583                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1584                    continue;
1585                };
1586                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1587                    continue;
1588                };
1589
1590                match self.forwarding.release_client_route(
1591                    route.goodbye_target.connection_id,
1592                    route.goodbye_target.channel,
1593                    route.goodbye_target.epoch,
1594                ) {
1595                    Ok(RouteRelease::Removed(module_goodbye)) => {
1596                        warn!(
1597                            opening_module_id,
1598                            target_module_id,
1599                            capability,
1600                            "force-closing route because an attested capability deny edge now matches"
1601                        );
1602                        goodbyes.push(module_goodbye);
1603                        goodbyes.push(route.goodbye_target.clone());
1604                        closed_routes.push(route);
1605                    }
1606                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1607                    Err(err) => warn!(
1608                        opening_module_id,
1609                        target_module_id,
1610                        capability,
1611                        error = %err,
1612                        "failed to force-close capability-denied route"
1613                    ),
1614                }
1615            }
1616
1617            if closed_routes.is_empty() {
1618                continue;
1619            }
1620            send_route_control_pushes(
1621                &self.forwarding,
1622                closed_routes,
1623                ClientControlPush::RouteClosed {
1624                    module_id: target_module_id,
1625                    channels: Vec::new(),
1626                    reason: RouteCloseReason::CapabilityDenied,
1627                    drained: false,
1628                    abandoned: 0,
1629                    excluded_subscriptions: 0,
1630                    terminal: Some(false),
1631                },
1632            );
1633            // route.closed records the reason; the route GOODBYE ends the
1634            // client's handle and settles any calls still pending on it.
1635            self.emit_route_goodbyes(goodbyes);
1636        }
1637    }
1638
1639    /// Why a registered module is not accepting new route binds, or `None` when
1640    /// it is. This is the module's effective readiness: its declared readiness
1641    /// first, then every `need: required` capability it declares evaluating to
1642    /// `provided`. `route.open` and `catalog.list` both read it here so the
1643    /// catalog never reports a module routable that `route.open` would refuse.
1644    fn not_ready_reason(
1645        &self,
1646        registration: &crate::registry::ModuleRegistration,
1647    ) -> Option<NotReadyReason> {
1648        if !registration.ready {
1649            return Some(NotReadyReason {
1650                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1651                capability: None,
1652            });
1653        }
1654        self.first_unprovided_required_capability(registration)
1655            .map(|capability| NotReadyReason {
1656                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1657                capability: Some(capability),
1658            })
1659    }
1660
1661    /// The lexicographically first capability this registration declares
1662    /// `need: required` whose evaluator verdict is not `provided`.
1663    ///
1664    /// The verdicts are the capability evaluator's own; nothing here decides
1665    /// what "provided" means. The evaluator counts a capability provided as
1666    /// soon as a module claiming it has REGISTERED, not once that module is
1667    /// ready. That distinction is what keeps two modules that require each
1668    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1669    /// is ready", each would wait for the other to become ready first and
1670    /// neither ever would. Do not tighten it to readiness.
1671    ///
1672    /// A required capability with no verdict at all means this registration's
1673    /// HELLO or catalog.update landed after the last recompute; recompute once
1674    /// rather than let a missing verdict read as either answer. If it is still
1675    /// missing (the recompute itself failed) the capability counts as
1676    /// unprovided: the refusal is retryable, and routing a module whose
1677    /// required provider is unknown is the outcome this check exists to stop.
1678    fn first_unprovided_required_capability(
1679        &self,
1680        registration: &crate::registry::ModuleRegistration,
1681    ) -> Option<String> {
1682        let required = registration
1683            .manifest
1684            .capabilities
1685            .iter()
1686            .flat_map(|declarations| declarations.requires.iter())
1687            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1688            .map(|requirement| requirement.capability.as_str())
1689            .collect::<BTreeSet<_>>();
1690        if required.is_empty() {
1691            return None;
1692        }
1693        let module_id = registration.manifest.module_id.as_str();
1694        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1695        if required
1696            .iter()
1697            .any(|capability| verdict(capability).is_none())
1698        {
1699            self.refresh_capability_requirements();
1700        }
1701        required
1702            .into_iter()
1703            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1704            .map(str::to_string)
1705    }
1706
1707    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1708        self.capability_evaluator
1709            .statuses()
1710            .into_iter()
1711            .map(capability_requirement_status)
1712            .collect()
1713    }
1714
1715    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1716    /// registration-release watch. The signal is what the supervisor waits on
1717    /// before spawning a replacement, so it must only fire once forwarding
1718    /// teardown is also done (see [`Self::cleanup_connection`] /
1719    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1720    /// admitted but its forwarding endpoint could not be installed.
1721    fn deregister_connection(
1722        &self,
1723        connection_id: ConnectionId,
1724        reason: RegistrationEndReason,
1725    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1726        self.registry
1727            .deregister_connection_with_reason(connection_id, reason)
1728    }
1729
1730    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1731        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1732            return None;
1733        }
1734        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1735            parse_client_control_request(&frame.body)
1736        else {
1737            return None;
1738        };
1739        Some(target_module_id(&target).to_string())
1740    }
1741
1742    pub(crate) fn route_open_capacity_refusal(
1743        &self,
1744        ctx: &RouteCtx,
1745        frame: &Frame,
1746        target_module_id: &str,
1747        in_flight: usize,
1748        limit: usize,
1749    ) -> Result<Frame, RouterError> {
1750        // Sent before the request reaches `handle_route_open`, so the caller
1751        // fields come from the raw frame here.
1752        let caller = self.route_open_caller_from_frame(frame);
1753        self.route_open_admission_refusal_frame(
1754            ctx,
1755            &caller,
1756            frame,
1757            target_module_id,
1758            "open_admission_full",
1759            (in_flight, limit),
1760            format!(
1761                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1762            ),
1763        )
1764    }
1765
1766    fn route_open_target_capacity_refusal(
1767        &self,
1768        ctx: &RouteCtx,
1769        caller: &RouteOpenCaller,
1770        frame: &Frame,
1771        target_module_id: &str,
1772        in_flight: usize,
1773    ) -> Result<Frame, RouterError> {
1774        self.route_open_admission_refusal_frame(
1775            ctx,
1776            caller,
1777            frame,
1778            target_module_id,
1779            "target_binds_full",
1780            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1781            format!(
1782                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1783            ),
1784        )
1785    }
1786
1787    /// Admission pressure clears as existing binds settle, so its refusal must
1788    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1789    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1790    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1791    /// cannot currently reach its target; `module_timeout` would falsely claim
1792    /// that a wait expired. A new, cleaner code would be terminal to deployed
1793    /// clients, so it requires a client-tolerance rollout before daemon emission.
1794    #[allow(clippy::too_many_arguments)]
1795    fn route_open_admission_refusal_frame(
1796        &self,
1797        ctx: &RouteCtx,
1798        caller: &RouteOpenCaller,
1799        frame: &Frame,
1800        target_module_id: &str,
1801        reason: &'static str,
1802        (in_flight, limit): (usize, usize),
1803        message: impl Into<String>,
1804    ) -> Result<Frame, RouterError> {
1805        let code = error_codes::TARGET_UNAVAILABLE;
1806        self.counters.increment_route_open_refused(code);
1807        info!(
1808            target: "control",
1809            code,
1810            reason,
1811            module_id = ?target_module_id,
1812            connection_id = ctx.connection_id.get(),
1813            principal = %caller.principal,
1814            scope = %caller.scope,
1815            harness = %caller.harness,
1816            in_flight,
1817            limit,
1818            "route.open refused"
1819        );
1820        control_error_frame(frame, code, message.into())
1821    }
1822
1823    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1824    ///
1825    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1826    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1827    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1828    #[cfg(test)]
1829    pub fn handle_control(
1830        &self,
1831        connection_id: ConnectionId,
1832        frame: Frame,
1833    ) -> Result<Vec<Frame>, RouterError> {
1834        match frame.header.ty {
1835            FrameType::Ping => Ok(vec![pong(&frame)?]),
1836            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1837            FrameType::Goodbye => self.handle_goodbye(connection_id),
1838            ty => Ok(vec![control_error_frame(
1839                &frame,
1840                "unsupported_control_frame",
1841                format!("unsupported channel-0 frame {ty:?}"),
1842            )?]),
1843        }
1844    }
1845
1846    pub async fn handle_control_frame(
1847        &self,
1848        ctx: &RouteCtx,
1849        frame: Frame,
1850    ) -> Result<Vec<Frame>, RouterError> {
1851        self.handle_control_frame_timed(ctx, frame, None).await
1852    }
1853
1854    #[cfg(any(test, feature = "test-support"))]
1855    pub fn with_test_operator_app_policy(
1856        mut self,
1857        policy: crate::operator_apps::TestOperatorAppPolicy,
1858    ) -> Self {
1859        self.operator_app_policy = policy.into();
1860        self
1861    }
1862
1863    /// Inject the Windows signature check for tests. The injected check
1864    /// answers the verified version string or the rejection reason; the
1865    /// daemon enforces the floor itself on the returned version.
1866    #[cfg(all(windows, any(test, feature = "test-support")))]
1867    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1868        self.windows_verify = Some(verify);
1869        self
1870    }
1871
1872    /// Record which pids the daemon would flash for `already_running`.
1873    #[cfg(all(windows, any(test, feature = "test-support")))]
1874    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1875        self.windows_flash = Some(flash);
1876        self
1877    }
1878
1879    /// Inject a tracking failure for tests: makes `track_launched_process`
1880    /// fail so a test can prove a launch that cannot be tracked is stopped,
1881    /// never left running.
1882    #[cfg(all(windows, any(test, feature = "test-support")))]
1883    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1884        self.windows_track = Some(track);
1885        self
1886    }
1887
1888    pub(crate) fn record_captured_peer(
1889        &self,
1890        connection: ConnectionId,
1891        peer: subc_os::local_peer::CapturedPeer,
1892    ) {
1893        self.record_observed_peer(connection, peer.observation().clone());
1894        #[cfg(windows)]
1895        if let Some(stamp) = self.stamp_windows_launch(&peer) {
1896            self.operator_apps
1897                .lock()
1898                .unwrap_or_else(|p| p.into_inner())
1899                .insert(connection, stamp);
1900        }
1901        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1902            self.operator_apps
1903                .lock()
1904                .unwrap_or_else(|p| p.into_inner())
1905                .insert(connection, stamp);
1906        }
1907    }
1908
1909    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1910        self.local_connections
1911            .lock()
1912            .unwrap_or_else(|p| p.into_inner())
1913            .insert(connection);
1914    }
1915
1916    // Keep the connection-level delivery decision shared with route.bind. The
1917    // target's capability opt-in is separate from recognition of the caller.
1918    fn connection_operator_app(
1919        &self,
1920        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1921    ) -> Option<subc_protocol::peer::OperatorApp> {
1922        stamp
1923            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1924            .map(|stamp| stamp.app.clone())
1925    }
1926
1927    fn handle_connection_describe(
1928        &self,
1929        ctx: &RouteCtx,
1930        frame: Frame,
1931    ) -> Result<Vec<Frame>, RouterError> {
1932        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1933            return Ok(vec![control_error_frame(
1934                &frame,
1935                "invalid_control_body",
1936                "malformed connection.describe body",
1937            )?]);
1938        }
1939        let local = self
1940            .local_connections
1941            .lock()
1942            .unwrap_or_else(|p| p.into_inner())
1943            .contains(&ctx.connection_id);
1944        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1945        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1946        Ok(vec![control_response_body_frame(
1947            &frame,
1948            &subc_control::ConnectionDescribeReply::new(local, operator_app),
1949            ops::CONNECTION_DESCRIBE,
1950        )?])
1951    }
1952
1953    pub(crate) fn record_observed_peer(
1954        &self,
1955        connection: ConnectionId,
1956        peer: subc_os::local_peer::ObservedPeer,
1957    ) {
1958        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1959        let peer = match peer {
1960            OsPeer::Unobserved => ObservedPeer::unobserved(),
1961            OsPeer::Observed(peer) => {
1962                let observed = match peer.user {
1963                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1964                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1965                };
1966                observed
1967                    .with_exe(
1968                        peer.exe
1969                            .and_then(|path| path.into_os_string().into_string().ok()),
1970                    )
1971                    .with_signing_id(peer.signing_id)
1972            }
1973        };
1974        self.observed_peers
1975            .lock()
1976            .unwrap_or_else(|p| p.into_inner())
1977            .insert(connection, peer);
1978    }
1979
1980    pub(crate) fn record_caller_endpoints(
1981        &self,
1982        connection: ConnectionId,
1983        endpoints: Option<(SocketAddr, SocketAddr)>,
1984    ) {
1985        let mut callers = self
1986            .caller_endpoints
1987            .lock()
1988            .unwrap_or_else(|p| p.into_inner());
1989        callers.remove(&connection);
1990        if let Some(endpoints) = endpoints {
1991            callers.insert(connection, endpoints);
1992        }
1993    }
1994
1995    #[cfg(test)]
1996    pub(crate) fn with_caller_lookup_for_test(
1997        mut self,
1998        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1999    ) -> Self {
2000        self.caller_lookup = Some(Arc::new(lookup));
2001        self
2002    }
2003
2004    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2005        let endpoints = self
2006            .caller_endpoints
2007            .lock()
2008            .unwrap_or_else(|p| p.into_inner())
2009            .get(&connection)
2010            .copied();
2011        #[cfg(test)]
2012        let lookup = self.caller_lookup.clone();
2013        tokio::task::spawn_blocking(move || {
2014            #[cfg(test)]
2015            if let Some(lookup) = lookup {
2016                return lookup(endpoints);
2017            }
2018            endpoints
2019                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2020                .unwrap_or_default()
2021        })
2022        .await
2023        .unwrap_or_default()
2024    }
2025
2026    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2027        frame.header.channel == 0
2028            && frame.header.ty == FrameType::Request
2029            && self
2030                .forwarding
2031                .module_endpoint_for_connection(ctx.connection_id)
2032                .is_ok_and(|endpoint| endpoint.is_none())
2033            && parse_client_control_request(&frame.body)
2034                .ok()
2035                .is_some_and(|request| privileged_operation(&request).is_some())
2036    }
2037
2038    pub(crate) async fn handle_control_frame_timed(
2039        &self,
2040        ctx: &RouteCtx,
2041        frame: Frame,
2042        dispatch_started_at: Option<StdInstant>,
2043    ) -> Result<Vec<Frame>, RouterError> {
2044        match frame.header.ty {
2045            FrameType::Ping => Ok(vec![pong(&frame)?]),
2046            FrameType::Hello => {
2047                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2048            }
2049            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2050            FrameType::Cancel => {
2051                // Channel-0 cancellation also fences pending declarations;
2052                // successful cancellation has no ordinary reply.
2053                if self
2054                    .agent_declarations
2055                    .cancel(ctx.connection_id, frame.header.corr)
2056                    || self
2057                        .forwarding
2058                        .operator_confirms()
2059                        .cancel(ctx.connection_id, frame.header.corr)
2060                    || self
2061                        .supervisor
2062                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2063                    || self.cancel_activity_subscription(ctx, &frame)?
2064                {
2065                    Ok(Vec::new())
2066                } else {
2067                    Ok(vec![control_error_frame(
2068                        &frame,
2069                        "unknown_subscription",
2070                        "no subscription has this correlation id",
2071                    )?])
2072                }
2073            }
2074            FrameType::Request => {
2075                // This additive operation is not part of the existing exhaustive
2076                // module-control enum. Probe the op before decoding that enum.
2077                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2078                if op
2079                    .as_ref()
2080                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2081                {
2082                    return self.handle_activity_record(ctx, frame).await;
2083                }
2084                if op.as_ref().is_some_and(|probe| {
2085                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2086                }) {
2087                    return self.handle_activity_read(ctx, frame).await;
2088                }
2089                if op
2090                    .as_ref()
2091                    .is_some_and(|probe| probe.op == "operator.confirm")
2092                {
2093                    return self.handle_operator_confirm(ctx, frame);
2094                }
2095                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2096                    return self
2097                        .observe_control_wait(
2098                            ctx,
2099                            None,
2100                            ops::APP_OPEN,
2101                            self.handle_app_open(ctx, frame),
2102                        )
2103                        .await;
2104                }
2105                if op
2106                    .as_ref()
2107                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2108                {
2109                    return self.handle_connection_describe(ctx, frame);
2110                }
2111                if op
2112                    .as_ref()
2113                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2114                {
2115                    return self.handle_scope_served_by(ctx.connection_id, frame);
2116                }
2117                if self
2118                    .forwarding
2119                    .module_endpoint_for_connection(ctx.connection_id)
2120                    .map_err(RouterError::Forwarding)?
2121                    .is_some()
2122                {
2123                    if !is_known_module_request_op(&frame.body) {
2124                        return Ok(vec![control_error_frame(
2125                            &frame,
2126                            "unsupported_control_frame",
2127                            "module-originated channel-0 REQUEST is not supported",
2128                        )?]);
2129                    }
2130                    let request = match parse_module_control_request_from_module(&frame.body) {
2131                        Ok(request) => request,
2132                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2133                            return Ok(vec![control_error_frame(
2134                                &frame,
2135                                "unsupported_control_frame",
2136                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2137                            )?])
2138                        }
2139                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2140                            return Ok(vec![control_error_frame(
2141                                &frame,
2142                                "invalid_control_body",
2143                                format!("malformed module control body: {err}"),
2144                            )?])
2145                        }
2146                    };
2147                    let op = module_control_request_op(&request);
2148                    let corr = frame.header.corr;
2149                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2150                    if let ModuleControlRequestFromModule::AgentCommands {
2151                        started,
2152                        finished,
2153                        known,
2154                    } = request
2155                    {
2156                        return self
2157                            .agent_declarations
2158                            .enqueue(
2159                                self,
2160                                ctx,
2161                                frame,
2162                                crate::resource_totals::Declaration {
2163                                    started,
2164                                    finished,
2165                                    known,
2166                                },
2167                            )
2168                            .await;
2169                    }
2170                    let result =
2171                        self.handle_module_control_request(ctx.connection_id, frame, request);
2172                    log_slow_control_dispatch(
2173                        dispatch_started_at,
2174                        op,
2175                        ctx.connection_id,
2176                        corr,
2177                        None,
2178                    );
2179                    return result;
2180                }
2181
2182                if is_known_module_request_op(&frame.body) {
2183                    let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2184                        .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2185                    return Ok(vec![control_error_frame(
2186                        &frame,
2187                        if is_agents { "not_supervised" } else { "not_registered" },
2188                        "module-originated control requests require the module's own registered connection",
2189                    )?]);
2190                }
2191
2192                let request = match parse_client_control_request(&frame.body) {
2193                    Ok(request) => request,
2194                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2195                        return Ok(vec![control_error_frame(
2196                            &frame,
2197                            "unknown_control_op",
2198                            format!("unknown client control op: {err}"),
2199                        )?])
2200                    }
2201                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2202                        return Ok(vec![control_error_frame(
2203                            &frame,
2204                            "invalid_control_body",
2205                            format!("malformed client control body: {err}"),
2206                        )?])
2207                    }
2208                };
2209                let op = client_control_request_op(&request);
2210                let corr = frame.header.corr;
2211                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2212                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2213                #[cfg(test)]
2214                if let Some(delay) = self.control_dispatch_delay {
2215                    tokio::time::sleep(delay).await;
2216                }
2217                let result = self
2218                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2219                    .await;
2220                if let Some(timings) = route_open_timings.as_mut() {
2221                    timings.finish();
2222                }
2223                log_slow_control_dispatch(
2224                    dispatch_started_at,
2225                    op,
2226                    ctx.connection_id,
2227                    corr,
2228                    route_open_timings.as_ref(),
2229                );
2230                result
2231            }
2232            FrameType::Push => {
2233                let Some(endpoint) = self
2234                    .forwarding
2235                    .module_endpoint_for_connection(ctx.connection_id)
2236                    .map_err(RouterError::Forwarding)?
2237                else {
2238                    return Ok(vec![control_error_frame(
2239                        &frame,
2240                        "unsupported_control_frame",
2241                        "client-originated channel-0 PUSH is not supported",
2242                    )?]);
2243                };
2244                self.handle_status_update(endpoint, frame)
2245            }
2246            FrameType::Response | FrameType::Error
2247                if self
2248                    .forwarding
2249                    .module_endpoint_for_connection(ctx.connection_id)
2250                    .map_err(RouterError::Forwarding)?
2251                    .is_some() =>
2252            {
2253                self.handle_module_relay_response(ctx.connection_id, frame)
2254            }
2255            ty => Ok(vec![control_error_frame(
2256                &frame,
2257                "unsupported_control_frame",
2258                format!("unsupported channel-0 frame {ty:?}"),
2259            )?]),
2260        }
2261    }
2262
2263    pub fn cleanup_connection(
2264        &self,
2265        connection_id: ConnectionId,
2266    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2267        self.local_connections
2268            .lock()
2269            .unwrap_or_else(|p| p.into_inner())
2270            .remove(&connection_id);
2271        self.caller_endpoints
2272            .lock()
2273            .unwrap_or_else(|p| p.into_inner())
2274            .remove(&connection_id);
2275        self.build_class_warned
2276            .lock()
2277            .unwrap_or_else(|p| p.into_inner())
2278            .remove(&connection_id);
2279        self.observed_peers
2280            .lock()
2281            .unwrap_or_else(|p| p.into_inner())
2282            .remove(&connection_id);
2283        self.operator_apps
2284            .lock()
2285            .unwrap_or_else(|p| p.into_inner())
2286            .remove(&connection_id);
2287        self.forwarding
2288            .operator_confirms()
2289            .connection_closed(connection_id);
2290        self.cleanup_connection_with_end_reason(
2291            connection_id,
2292            RegistrationEndReason::ConnectionClosed,
2293        )
2294    }
2295
2296    fn cleanup_connection_with_end_reason(
2297        &self,
2298        connection_id: ConnectionId,
2299        requested_reason: RegistrationEndReason,
2300    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2301        self.agent_declarations.close(connection_id);
2302        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2303            self.registry
2304                .get_module_by_connection(connection_id)?
2305                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2306                .and_then(|module| module.registration_end_reason().ok().flatten())
2307                .unwrap_or(requested_reason)
2308        } else {
2309            requested_reason
2310        };
2311        let crash_closed = self
2312            .registry
2313            .get_module_by_connection(connection_id)?
2314            .and_then(|registration| {
2315                self.forwarding
2316                    .module_endpoint_for_connection(connection_id)
2317                    .ok()
2318                    .flatten()
2319                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2320                    .map(|routes| (registration.manifest.module_id, routes))
2321            });
2322        let crash_closed = crash_closed.map(|(module_id, routes)| {
2323            let terminal = match self.supervisor.get(&module_id) {
2324                None => false,
2325                Some(module) => match module.will_recover_after_connection_loss() {
2326                    Ok(will_recover) => !will_recover,
2327                    Err(err) => {
2328                        warn!(
2329                            %module_id,
2330                            error = %err,
2331                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2332                        );
2333                        false
2334                    }
2335                },
2336            };
2337            // The forwarding table gates all providers at the start of daemon
2338            // shutdown, before their connections are closed. An ordinary
2339            // module disconnect still reports crash if that gate is not set.
2340            let (reason, drained) = match self
2341                .forwarding
2342                .daemon_drain_result_for_connection(connection_id)
2343            {
2344                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2345                Ok(None) => (RouteCloseReason::Crash, false),
2346                Err(err) => {
2347                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2348                    (RouteCloseReason::Crash, false)
2349                }
2350            };
2351            (module_id, routes, reason, drained, terminal)
2352        });
2353        let registrations = self.deregister_connection(connection_id, end_reason);
2354        let cleanup = if crash_closed.is_some() {
2355            self.forwarding.cleanup_connection_counted(connection_id)
2356        } else {
2357            // A module connection's teardown needs the count of abandoned
2358            // route.bind relays for its route.closed notice below. Any other
2359            // connection, such as a client's, sends no such notice and needs
2360            // only its routes released, so it uses the route-only wrapper and
2361            // reports zero.
2362            self.forwarding
2363                .cleanup_connection(connection_id)
2364                .map(|released| crate::forwarding::ConnectionCleanup {
2365                    released,
2366                    abandoned_relays: 0,
2367                })
2368        };
2369        // The route.closed push waits for forwarding teardown because only
2370        // teardown knows how many pending route.bind relays it aborted. It still
2371        // goes out before the GOODBYEs for the released routes, and its targets
2372        // were captured above, before teardown removed those routes.
2373        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2374            let abandoned = cleanup
2375                .as_ref()
2376                .map_or(0, |cleanup| cleanup.abandoned_relays);
2377            send_route_control_pushes(
2378                &self.forwarding,
2379                routes,
2380                ClientControlPush::RouteClosed {
2381                    module_id,
2382                    channels: Vec::new(),
2383                    reason,
2384                    drained,
2385                    abandoned,
2386                    excluded_subscriptions: 0,
2387                    terminal: Some(terminal),
2388                },
2389            );
2390        }
2391        if let Ok(cleanup) = cleanup {
2392            self.emit_route_goodbyes(cleanup.released);
2393        }
2394        // Signal the registration-release watch only now that BOTH registry and
2395        // forwarding teardown are done, so a supervisor waiting to spawn a
2396        // replacement never observes release while old routes still exist.
2397        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2398            crate::supervise::notify_registration_release();
2399            self.capability_evaluator.wake_deadline_loop();
2400            self.refresh_capability_requirements();
2401        }
2402        self.supervisor.remove_spawn_subscribers(connection_id);
2403        self.activity_streams
2404            .lock()
2405            .unwrap_or_else(|p| p.into_inner())
2406            .retain(|(id, _), task| {
2407                if *id == connection_id {
2408                    task.abort();
2409                    false
2410                } else {
2411                    true
2412                }
2413            });
2414        // Sync authority dies with its connection, so the owner's next
2415        // connection can take it; the owner's scopes stay as they are.
2416        self.hello_launch_nonces
2417            .lock()
2418            .unwrap_or_else(|poisoned| poisoned.into_inner())
2419            .forget(connection_id);
2420        self.scopes
2421            .write()
2422            .unwrap_or_else(|poisoned| poisoned.into_inner())
2423            .release_connection(connection_id);
2424        registrations
2425    }
2426
2427    pub(crate) fn handle_route_goodbye(
2428        &self,
2429        connection_id: ConnectionId,
2430        route_channel: u16,
2431        route_epoch: u32,
2432    ) -> Result<bool, RouterError> {
2433        debug!(
2434            connection_id = connection_id.get(),
2435            route_channel, route_epoch, "handling route GOODBYE"
2436        );
2437        let RouteRelease::Removed(released_route) = self
2438            .forwarding
2439            .release_client_route(connection_id, route_channel, route_epoch)
2440            .map_err(RouterError::Forwarding)?
2441        else {
2442            return Ok(false);
2443        };
2444        self.emit_route_goodbyes(vec![released_route]);
2445        Ok(true)
2446    }
2447
2448    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2449        for released in released_routes {
2450            let frame = match Frame::build_with_version(
2451                released.negotiated_ver,
2452                FrameType::Goodbye,
2453                control_flags(),
2454                released.channel,
2455                released.epoch,
2456                0,
2457                Vec::new(),
2458            ) {
2459                Ok(frame) => frame,
2460                Err(err) => {
2461                    warn!(
2462                        route_channel = released.channel,
2463                        error = %err,
2464                        "failed to build route GOODBYE frame"
2465                    );
2466                    continue;
2467                }
2468            };
2469            if !released.close_on_delivery_failure() {
2470                crate::forwarding::send_module_route_goodbye(
2471                    &self.counters,
2472                    &released.sink,
2473                    frame,
2474                    released.module_id.as_deref(),
2475                    "client route released",
2476                );
2477                continue;
2478            }
2479            if let Err(err) = released.sink.try_send(frame) {
2480                warn!(
2481                    target_connection_id = released.connection_id.get(),
2482                    route_channel = released.channel,
2483                    error = %err,
2484                    "route GOODBYE was not delivered to client; closing target connection"
2485                );
2486                if self
2487                    .forwarding
2488                    .escalate_client_delivery_failure(
2489                        released.connection_id,
2490                        released.channel,
2491                        released.epoch,
2492                        CloseReason::new(
2493                            "route_goodbye_delivery_failed",
2494                            format!(
2495                                "failed to enqueue route GOODBYE for channel {}: {err}",
2496                                released.channel
2497                            ),
2498                        ),
2499                        crate::forwarding::UndeliveredFrame {
2500                            module_id: released.module_id.as_deref(),
2501                            sink: &released.sink,
2502                        },
2503                    )
2504                    .unwrap_or(false)
2505                {
2506                    self.counters.increment_goodbye_relay_client_failed();
2507                }
2508            }
2509        }
2510    }
2511
2512    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2513    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2514    /// own commit failed after the module had already accepted). Without this, a
2515    /// module that accepts late keeps a binding subc has torn down, so a later
2516    /// frame on that module channel could misdeliver if the channel is reused.
2517    ///
2518    /// Never closes the shared module connection on failure: a dropped notification
2519    /// only wastes a bounded amount of warm module-side state, which the module's
2520    /// own idle reaper reclaims. Only call this once the route.bind relay was
2521    /// actually enqueued to the module — if the relay send itself failed, the
2522    /// module never created a binding and there is nothing to tear down.
2523    fn send_abandoned_route_bind_goodbye(
2524        &self,
2525        module_sink: &crate::FrameSink,
2526        negotiated_ver: u8,
2527        module_channel: u16,
2528        module_epoch: u32,
2529    ) {
2530        let frame = match Frame::build_with_version(
2531            negotiated_ver,
2532            FrameType::Goodbye,
2533            control_flags(),
2534            module_channel,
2535            module_epoch,
2536            0,
2537            Vec::new(),
2538        ) {
2539            Ok(frame) => frame,
2540            Err(err) => {
2541                warn!(
2542                    route_channel = module_channel,
2543                    error = %err,
2544                    "failed to build GOODBYE for abandoned route.bind"
2545                );
2546                return;
2547            }
2548        };
2549        crate::forwarding::send_module_route_goodbye(
2550            &self.counters,
2551            module_sink,
2552            frame,
2553            None,
2554            "abandoned route.bind",
2555        );
2556    }
2557
2558    fn handle_hello(
2559        &self,
2560        connection_id: ConnectionId,
2561        sink: Option<crate::FrameSink>,
2562        frame: Frame,
2563    ) -> Result<Vec<Frame>, RouterError> {
2564        debug!(
2565            connection_id = connection_id.get(),
2566            corr = frame.header.corr,
2567            "handling HELLO"
2568        );
2569        // A module connection has one identity for its entire lifetime. A second
2570        // registration would leave the old registry owner behind while replacing
2571        // its forwarding endpoint and launch nonce.
2572        if self
2573            .registry
2574            .get_module_by_connection(connection_id)
2575            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2576            .is_some()
2577        {
2578            return Ok(vec![control_error_frame(
2579                &frame,
2580                "invalid_hello",
2581                "connection is already registered as a module",
2582            )?]);
2583        }
2584        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2585            Ok(value) => value,
2586            Err(err) => {
2587                return Ok(vec![control_error_frame(
2588                    &frame,
2589                    "invalid_hello",
2590                    format!("malformed HELLO body: {err}"),
2591                )?])
2592            }
2593        };
2594        if let Err(err) = validate_hello_display_name(&hello_value) {
2595            return Ok(vec![control_error_frame(
2596                &frame,
2597                "invalid_manifest",
2598                err.to_string(),
2599            )?]);
2600        }
2601        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2602            return Ok(vec![control_error_frame(
2603                &frame,
2604                "invalid_capability_grammar",
2605                err.to_string(),
2606            )?]);
2607        }
2608        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2609            return Ok(vec![control_error_frame(
2610                &frame,
2611                "invalid_manifest",
2612                err.to_string(),
2613            )?]);
2614        }
2615        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2616            return Ok(vec![control_error_frame(
2617                &frame,
2618                "invalid_event_declaration",
2619                err.to_string(),
2620            )?]);
2621        }
2622        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2623            return Ok(vec![control_error_frame(
2624                &frame,
2625                "invalid_manifest",
2626                err.to_string(),
2627            )?]);
2628        }
2629        if let Some(provenance) = hello_value
2630            .get("manifest")
2631            .and_then(|manifest| manifest.get("provenance"))
2632        {
2633            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2634                return Ok(vec![control_error_frame(
2635                    &frame,
2636                    "invalid_manifest",
2637                    format!("malformed manifest provenance: {err}"),
2638                )?]);
2639            }
2640        }
2641        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2642            let module_id = hello_value["manifest"]["module_id"]
2643                .as_str()
2644                .unwrap_or("<unknown>");
2645            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2646        }
2647        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2648            let module_id = hello_value["manifest"]["module_id"]
2649                .as_str()
2650                .unwrap_or("<unknown>");
2651            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2652        }
2653        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2654            let module_id = hello_value["manifest"]["module_id"]
2655                .as_str()
2656                .unwrap_or("<unknown>");
2657            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2658        }
2659        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2660            Ok(hello) => hello,
2661            Err(err) => {
2662                return Ok(vec![control_error_frame(
2663                    &frame,
2664                    "invalid_hello",
2665                    format!("malformed HELLO body: {err}"),
2666                )?])
2667            }
2668        };
2669
2670        if hello.protocol_ver != hello.manifest.protocol_ver {
2671            return Ok(vec![control_error_frame(
2672                &frame,
2673                "invalid_manifest",
2674                format!(
2675                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2676                    hello.protocol_ver, hello.manifest.protocol_ver
2677                ),
2678            )?]);
2679        }
2680
2681        if hello.manifest.module_id.trim().is_empty() {
2682            return Ok(vec![control_error_frame(
2683                &frame,
2684                "invalid_manifest",
2685                "manifest module_id must not be empty",
2686            )?]);
2687        }
2688
2689        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2690            Ok(negotiated_ver) => negotiated_ver,
2691            Err(message) => {
2692                return Ok(vec![control_error_frame(
2693                    &frame,
2694                    "version_unsupported",
2695                    message,
2696                )?])
2697            }
2698        };
2699
2700        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2701        // swap is open for this id, the only HELLO admitted as a second process
2702        // is the one carrying the candidate's launch nonce (the swap token), and
2703        // it registers into the candidate slot rather than being refused as a
2704        // duplicate. Run after the reserved gate, a reserved module's candidate
2705        // would be refused `reserved_module` for presenting a nonce that gate
2706        // does not know. See `SupervisorHandle::swap_hello_admission`.
2707        let swap_admission = self
2708            .supervisor
2709            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2710        if swap_admission == SwapHelloAdmission::Refused {
2711            warn!(
2712                module_id = %hello.manifest.module_id,
2713                connection_id = connection_id.get(),
2714                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2715            );
2716            return Ok(vec![control_error_frame(
2717                &frame,
2718                "swap_token_invalid",
2719                format!(
2720                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2721                    hello.manifest.module_id
2722                ),
2723            )?]);
2724        }
2725        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2726
2727        // A swap candidate proves its identity with the candidate nonce above.
2728        // Other registrations receive this catalog fact only when reserved-module
2729        // policy required and accepted a nonce; unreserved registrations require
2730        // no nonce check.
2731        let verification = if swap_candidate {
2732            ReservedHelloVerification {
2733                nonce_verified: true,
2734                prefix_owner: None,
2735            }
2736        } else {
2737            match self.supervisor.reserved_hello_verification(
2738                &hello.manifest.module_id,
2739                hello.launch_nonce.as_deref(),
2740            ) {
2741                Ok(verified) => verified,
2742                Err(rejection) => {
2743                    let message = match rejection {
2744                        ReservedHelloRejection::Exact { module_id } => format!(
2745                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2746                        ),
2747                        ReservedHelloRejection::Prefix {
2748                            prefix,
2749                            owner_module_id,
2750                        } => format!(
2751                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2752                            hello.manifest.module_id
2753                        ),
2754                    };
2755                    return Ok(vec![control_error_frame(
2756                        &frame,
2757                        "reserved_module",
2758                        message,
2759                    )?]);
2760                }
2761            }
2762        };
2763
2764        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2765            &hello.manifest.module_id,
2766            hello.manifest.capabilities.as_ref(),
2767        );
2768        if let Some(refusal) = reserved_capability_refusals.first() {
2769            let capability = refusal.capability.clone();
2770            let bound_module = refusal.claimants[0].clone();
2771            log_duplicate_claim_events(reserved_capability_refusals);
2772            return Ok(vec![control_error_frame(
2773                &frame,
2774                "reserved_capability",
2775                format!(
2776                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2777                    capability, bound_module, hello.manifest.module_id
2778                ),
2779            )?]);
2780        }
2781
2782        // A connection that already opened client routes must not also register as
2783        // a module: cleanup would then release only one side and leak the other.
2784        if self
2785            .forwarding
2786            .connection_has_client_routes(connection_id)
2787            .map_err(RouterError::Forwarding)?
2788        {
2789            return Ok(vec![control_error_frame(
2790                &frame,
2791                "invalid_hello",
2792                "connection has open client routes and cannot also register as a module",
2793            )?]);
2794        }
2795
2796        // Kept for scope sync authority, which goes only to the connection that
2797        // presented the module's current launch nonce. Recorded before the
2798        // registration is attempted: a connection whose registration then fails
2799        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2800        self.hello_launch_nonces
2801            .lock()
2802            .unwrap_or_else(|poisoned| poisoned.into_inner())
2803            .record(connection_id, hello.launch_nonce.as_deref());
2804        let control_ops = effective_module_control_ops(hello.control_ops);
2805        // Built before anything is registered so an encoding failure leaves no
2806        // registry or forwarding state behind.
2807        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2808        if swap_candidate {
2809            return self.register_swap_candidate(
2810                connection_id,
2811                sink,
2812                &frame,
2813                hello.manifest,
2814                negotiated_ver,
2815                control_ops,
2816                hello_ack,
2817            );
2818        }
2819        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2820        // is refused, but `catalog.list` does not show the module and the
2821        // catalog generation does not move until it is published below, once
2822        // its forwarding endpoint exists. Listing it now would advertise a
2823        // module that `route.open` still refuses for having no forwarding
2824        // connection.
2825        let registration = match self
2826            .registry
2827            .register_unpublished_verified_with_control_ops(
2828                hello.manifest,
2829                negotiated_ver,
2830                connection_id,
2831                control_ops,
2832                verification.nonce_verified,
2833            ) {
2834            Ok(registration) => registration,
2835            Err(RegistryError::DuplicateModuleId { module_id }) => {
2836                return Ok(vec![control_error_frame(
2837                    &frame,
2838                    "duplicate_module_id",
2839                    format!(
2840                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2841                    ),
2842                )?])
2843            }
2844            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2845                return Ok(vec![control_error_frame(
2846                    &frame,
2847                    "invalid_module_id",
2848                    err.to_string(),
2849                )?])
2850            }
2851            Err(err) => {
2852                return Ok(vec![control_error_frame(
2853                    &frame,
2854                    "registry_error",
2855                    err.to_string(),
2856                )?])
2857            }
2858        };
2859        #[cfg(test)]
2860        self.hold_between_registry_and_forwarding();
2861
2862        let reply = if let Some(sink) = sink {
2863            // The forwarding table's module store is also the daemon-to-module
2864            // control-RPC lane, so every HELLO gets a live endpoint even when the
2865            // manifest has no routable provider role. Non-routable modules still
2866            // cannot receive route.bind in production: `handle_route_open` checks
2867            // the registry manifest with `target_has_required_role` before the
2868            // only production call to `begin_route_bind_relay_for` below that
2869            // route.open path. The remaining direct relay callers are unit tests
2870            // and benchmark harnesses that construct forwarding state explicitly.
2871            //
2872            // The HELLO_ACK is queued by the forwarding table itself, before the
2873            // endpoint becomes visible, and is NOT returned as a reply. A module
2874            // reads HELLO_ACK first and exits on anything else; a reply is only
2875            // written after this handler returns, by which time a route.open on
2876            // another connection could already have queued a route.bind request
2877            // for this module ahead of it.
2878            let concurrency = manifest_concurrency(&registration.manifest);
2879            if let Err(err) = self.forwarding.register_module_connection_acked(
2880                connection_id,
2881                registration.manifest.module_id.clone(),
2882                negotiated_ver,
2883                concurrency,
2884                sink,
2885                crate::forwarding::ModuleHelloAck {
2886                    frame: hello_ack,
2887                    verified_prefix_owner: verification.prefix_owner,
2888                },
2889            ) {
2890                // Forwarding registration failed, so there is no forwarding
2891                // state to tear down. Remove the registry entry, which was never
2892                // published and so never listed or counted in the catalog
2893                // generation, and signal the release watch directly.
2894                if matches!(
2895                    self.deregister_connection(
2896                        connection_id,
2897                        RegistrationEndReason::RegistrationFailed,
2898                    ),
2899                    Ok(r) if !r.is_empty()
2900                ) {
2901                    crate::supervise::notify_registration_release();
2902                }
2903                return Ok(vec![control_error_frame(
2904                    &frame,
2905                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2906                        "invalid_hello"
2907                    } else {
2908                        forwarding_error_code(&err)
2909                    },
2910                    err.to_string(),
2911                )?]);
2912            }
2913            Vec::new()
2914        } else {
2915            // No sink means no forwarding endpoint, so nothing can be routed
2916            // ahead of the ack; it goes out as the reply.
2917            vec![hello_ack]
2918        };
2919
2920        // The module is routable now (or, without a sink, as routable as it
2921        // will ever be), so the catalog may list it. Publishing bumps the
2922        // catalog generation in the same registry critical section that makes
2923        // the module listed.
2924        if !self
2925            .registry
2926            .publish_registration(connection_id)
2927            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2928        {
2929            debug!(
2930                module_id = %registration.manifest.module_id,
2931                connection_id = connection_id.get(),
2932                "registration was removed before it could be published"
2933            );
2934        }
2935
2936        // Exposure over assumption: Concurrency's serde default is pinned to the
2937        // pre-field behavior (ModuleManaged), so a management surface that is
2938        // genuinely Serial and just never declared it inherits concurrent
2939        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2940        // turns "no module has been bitten yet" into the checkable claim "no
2941        // module is exposed" -- one read of the boot log instead of a fleet
2942        // audit. Detected from the raw HELLO bytes because the serde default
2943        // deliberately erases the absent/declared distinction from the type.
2944        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2945            info!(
2946                module_id = %registration.manifest.module_id,
2947                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2948            );
2949        }
2950
2951        self.apply_registration_capabilities(&registration);
2952
2953        self.supervisor
2954            .record_activity_registration(&registration.manifest.module_id);
2955
2956        info!(
2957            module_id = %registration.manifest.module_id,
2958            module_version = %registration.manifest.module_version,
2959            negotiated_ver,
2960            routable_provider = manifest_provides_routable_role(&registration.manifest),
2961            connection_id = connection_id.get(),
2962            "module registered"
2963        );
2964
2965        Ok(reply)
2966    }
2967
2968    /// Register a HELLO the swap gate admitted into the candidate slot of the
2969    /// registry and of forwarding, where it is reachable over its own
2970    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2971    /// nothing routes to it until the supervisor cuts over.
2972    ///
2973    /// Registry first (unpublished), then forwarding, then publication, the
2974    /// same order as an ordinary HELLO; a forwarding failure removes the
2975    /// registry entry again. The swap supervisor treats a published, ready
2976    /// candidate as registered and promotes it, and promotion needs the
2977    /// candidate's forwarding endpoint, so the candidate is published only
2978    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2979    /// recompute capability verdicts or run the capability deny census
2980    /// (`enforce_capability_denies`): both are about the module in the active
2981    /// slot, which `route.open` can target. A candidate sits in the candidate
2982    /// slot, so it neither provides capabilities to other modules nor can be
2983    /// routed to until promotion moves it into the active slot.
2984    #[allow(clippy::too_many_arguments)]
2985    fn register_swap_candidate(
2986        &self,
2987        connection_id: ConnectionId,
2988        sink: Option<crate::FrameSink>,
2989        frame: &Frame,
2990        manifest: ModuleManifest,
2991        negotiated_ver: u8,
2992        control_ops: Vec<String>,
2993        hello_ack: Frame,
2994    ) -> Result<Vec<Frame>, RouterError> {
2995        let module_id = manifest.module_id.clone();
2996        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2997            manifest,
2998            negotiated_ver,
2999            connection_id,
3000            control_ops,
3001        ) {
3002            Ok(registration) => registration,
3003            Err(RegistryError::DuplicateModuleId { module_id }) => {
3004                return Ok(vec![control_error_frame(
3005                    frame,
3006                    "duplicate_module_id",
3007                    format!(
3008                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3009                    ),
3010                )?])
3011            }
3012            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3013                return Ok(vec![control_error_frame(
3014                    frame,
3015                    "invalid_module_id",
3016                    err.to_string(),
3017                )?])
3018            }
3019            Err(err) => {
3020                return Ok(vec![control_error_frame(
3021                    frame,
3022                    "registry_error",
3023                    err.to_string(),
3024                )?])
3025            }
3026        };
3027        #[cfg(test)]
3028        self.hold_between_registry_and_forwarding();
3029        let reply = if let Some(sink) = sink {
3030            // Same ordering as an ordinary HELLO: the forwarding table queues
3031            // the HELLO_ACK before the candidate endpoint is inserted, because
3032            // a module exits if its first frame after HELLO is anything else.
3033            let concurrency = manifest_concurrency(&registration.manifest);
3034            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3035                connection_id,
3036                module_id.clone(),
3037                negotiated_ver,
3038                concurrency,
3039                sink,
3040                hello_ack,
3041            ) {
3042                if matches!(
3043                    self.deregister_connection(
3044                        connection_id,
3045                        RegistrationEndReason::RegistrationFailed,
3046                    ),
3047                    Ok(r) if !r.is_empty()
3048                ) {
3049                    crate::supervise::notify_registration_release();
3050                }
3051                return Ok(vec![control_error_frame(
3052                    frame,
3053                    forwarding_error_code(&err),
3054                    err.to_string(),
3055                )?]);
3056            }
3057            Vec::new()
3058        } else {
3059            vec![hello_ack]
3060        };
3061        if !self
3062            .registry
3063            .publish_registration(connection_id)
3064            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3065        {
3066            debug!(
3067                module_id = %module_id,
3068                connection_id = connection_id.get(),
3069                "swap candidate registration was removed before it could be published"
3070            );
3071        }
3072        self.supervisor.mark_swap_candidate_admitted(&module_id);
3073        info!(
3074            module_id = %module_id,
3075            module_version = %registration.manifest.module_version,
3076            negotiated_ver,
3077            ready = registration.ready,
3078            connection_id = connection_id.get(),
3079            "swap candidate registered; not routable until cutover"
3080        );
3081        Ok(reply)
3082    }
3083
3084    fn build_hello_ack(
3085        &self,
3086        frame: &Frame,
3087        negotiated_ver: u8,
3088        module_id: &str,
3089    ) -> Result<Frame, RouterError> {
3090        let ack = ModuleHelloAckBody {
3091            negotiated_ver,
3092            subc_ops: module_subc_ops(),
3093            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3094            storage: self
3095                .storage_config
3096                .as_ref()
3097                .map(|cfg| cfg.descriptor_for(module_id)),
3098            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3099        };
3100        let body = serde_json::to_vec(&ack).map_err(|err| {
3101            RouterError::backend(
3102                0,
3103                frame.header.corr,
3104                format!("failed to encode HELLO_ACK: {err}"),
3105            )
3106        })?;
3107
3108        Frame::build_with_version(
3109            negotiated_ver,
3110            FrameType::HelloAck,
3111            control_flags(),
3112            0,
3113            0,
3114            frame.header.corr,
3115            body,
3116        )
3117        .map_err(RouterError::FrameBuild)
3118    }
3119
3120    async fn handle_client_control_request(
3121        &self,
3122        ctx: &RouteCtx,
3123        frame: Frame,
3124        request: ClientControlRequest,
3125        route_open_timings: &mut Option<RouteOpenTimings>,
3126    ) -> Result<Vec<Frame>, RouterError> {
3127        if let Some(operation) = privileged_operation(&request) {
3128            let caller = self.resolve_caller(ctx.connection_id).await;
3129            let op = client_control_request_op(&request);
3130            let text = prompt_text(operation, &caller);
3131            match self.privileged_operator_prompt {
3132                PrivilegedOperatorPrompt::Log => {
3133                    info!(target: "subc_daemon::privileged_prompt", operation = op,
3134                        target = ?operation, caller = %caller.description(), summary = %text,
3135                        "privileged request would prompt the operator");
3136                }
3137                PrivilegedOperatorPrompt::Enforce => {
3138                    let key =
3139                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3140                    let stamp = self
3141                        .operator_apps
3142                        .lock()
3143                        .unwrap_or_else(|p| p.into_inner())
3144                        .get(&ctx.connection_id)
3145                        .cloned();
3146                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3147                        &self.operator_app_policy,
3148                        &self.launch_table,
3149                        stamp.as_ref(),
3150                    )
3151                    .then(|| self.shortcut_context(stamp, None, None));
3152                    let outcome = self
3153                        .forwarding
3154                        .operator_confirms()
3155                        .approve_privileged_with_shortcut(
3156                            ctx,
3157                            &frame,
3158                            op,
3159                            text,
3160                            caller.description(),
3161                            key,
3162                            shortcut,
3163                        )
3164                        .await;
3165                    if outcome.code != "confirmed" {
3166                        return Ok(vec![outcome.frame(&frame)]);
3167                    }
3168                }
3169            }
3170        }
3171        match request {
3172            ClientControlRequest::DaemonShutdownIntent { intent } => {
3173                if intent.as_str() != Some("restart") {
3174                    return Ok(vec![control_error_frame(
3175                        &frame,
3176                        "invalid_params",
3177                        "intent must be 'restart'",
3178                    )?]);
3179                }
3180                let response = ClientControlResponse::DaemonShutdownIntent {
3181                    intent: "restart".to_owned(),
3182                    expires_at_ms: self.supervisor.announce_daemon_restart(),
3183                };
3184                Ok(vec![control_response_body_frame(
3185                    &frame,
3186                    &response,
3187                    "ClientControlResponse::DaemonShutdownIntent",
3188                )?])
3189            }
3190            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3191            ClientControlRequest::CatalogList { module_id } => {
3192                self.handle_catalog_list(frame, module_id)
3193            }
3194            ClientControlRequest::RouteOpen {
3195                target,
3196                identity,
3197                consumer_identity,
3198                consumer_capabilities,
3199                role_versions,
3200                busy_refusals,
3201                admission_facts,
3202                scope,
3203                ..
3204            } => {
3205                self.handle_route_open(
3206                    ctx,
3207                    frame,
3208                    RouteOpenRequest {
3209                        target,
3210                        identity,
3211                        consumer_identity,
3212                        consumer_capabilities,
3213                        role_versions,
3214                        busy_refusals,
3215                        admission_facts,
3216                        scope,
3217                    },
3218                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3219                )
3220                .await
3221            }
3222            ClientControlRequest::RoutePoll {
3223                route_channel,
3224                route_epoch,
3225                kind,
3226            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3227            ClientControlRequest::SupervisorList {} => {
3228                let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3229                    self.agent_declarations.refuse(
3230                        self,
3231                        ctx,
3232                        control_error_frame(
3233                            &frame,
3234                            "busy",
3235                            "supervisor list work is at capacity; retry",
3236                        )?,
3237                    );
3238                    return Ok(Vec::new());
3239                };
3240                // Filesystem-backed diagnostics must not hold up this connection's reader.
3241                let handler = self.clone();
3242                let sink = ctx.egress.clone();
3243                tokio::spawn(
3244                    async move {
3245                        let _permit = permit;
3246                        let result = handler.handle_supervisor_list(frame.clone()).await;
3247                        let replies = match result {
3248                            Ok(replies) => replies,
3249                            Err(error) => {
3250                                control_error_frame(&frame, "backend_error", error.to_string())
3251                                    .into_iter()
3252                                    .collect()
3253                            }
3254                        };
3255                        for reply in replies {
3256                            if !matches!(
3257                                tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3258                                    .await,
3259                                Ok(Ok(()))
3260                            ) {
3261                                break;
3262                            }
3263                        }
3264                    }
3265                    .with_current_subscriber(),
3266                );
3267                Ok(Vec::new())
3268            }
3269            ClientControlRequest::SupervisorSpawnSnapshot {} => {
3270                self.handle_supervisor_spawn_snapshot(frame)
3271            }
3272            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3273                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3274            }
3275            ClientControlRequest::SupervisorRestart {
3276                module_id,
3277                drain_timeout_ms,
3278            } => {
3279                self.log_supervisor_request_received(
3280                    ctx,
3281                    frame.header.corr,
3282                    ops::SUPERVISOR_RESTART,
3283                    Some(&module_id),
3284                    None,
3285                )?;
3286                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3287                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3288                    .await
3289            }
3290            ClientControlRequest::SupervisorSwap {
3291                module_id,
3292                ready_timeout_ms,
3293            } => {
3294                self.log_supervisor_request_received(
3295                    ctx,
3296                    frame.header.corr,
3297                    ops::SUPERVISOR_SWAP,
3298                    Some(&module_id),
3299                    None,
3300                )?;
3301                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3302                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3303                    .await
3304            }
3305            ClientControlRequest::SupervisorReload { module_id } => {
3306                self.log_supervisor_request_received(
3307                    ctx,
3308                    frame.header.corr,
3309                    ops::SUPERVISOR_RELOAD,
3310                    Some(&module_id),
3311                    None,
3312                )?;
3313                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3314                self.handle_supervisor_reload(frame, module_id, requester)
3315                    .await
3316            }
3317            ClientControlRequest::SupervisorRescan { preview } => {
3318                if !preview {
3319                    self.log_supervisor_request_received(
3320                        ctx,
3321                        frame.header.corr,
3322                        ops::SUPERVISOR_RESCAN,
3323                        None,
3324                        None,
3325                    )?;
3326                }
3327                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3328                let operation = self.handle_supervisor_rescan(
3329                    frame,
3330                    if preview {
3331                        RescanMode::Preview
3332                    } else {
3333                        RescanMode::Execute
3334                    },
3335                    requester,
3336                );
3337                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3338                    .await
3339            }
3340            ClientControlRequest::SupervisorRescanChecked {
3341                preview,
3342                candidate,
3343                expect,
3344            } => {
3345                self.observe_control_wait(
3346                    ctx,
3347                    None,
3348                    ops::SUPERVISOR_RESCAN,
3349                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3350                )
3351                .await
3352            }
3353            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3354                self.log_supervisor_request_received(
3355                    ctx,
3356                    frame.header.corr,
3357                    ops::SUPERVISOR_RELEASE_RESERVED,
3358                    Some(&module_id),
3359                    None,
3360                )?;
3361                self.handle_supervisor_release_reserved(frame, module_id)
3362                    .await
3363            }
3364            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3365                self.log_supervisor_request_received(
3366                    ctx,
3367                    frame.header.corr,
3368                    ops::SUPERVISOR_SET_ENABLED,
3369                    Some(&module_id),
3370                    Some(enabled),
3371                )?;
3372                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3373                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3374                    .await
3375            }
3376            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3377                self.handle_supervisor_health_probe(ctx, frame, module_id)
3378                    .await
3379            }
3380            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3381            ClientControlRequest::SupervisorRoutes { module_id } => {
3382                self.handle_supervisor_routes(frame, module_id)
3383            }
3384            ClientControlRequest::SupervisorProvenance { module_id } => {
3385                self.handle_supervisor_provenance(ctx, frame, module_id)
3386                    .await
3387            }
3388            ClientControlRequest::SupervisorStderrTail {
3389                module_id,
3390                max_lines,
3391                max_bytes,
3392            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3393            ClientControlRequest::SupervisorTerminals { module_id } => {
3394                self.handle_supervisor_terminals(ctx, frame, module_id)
3395                    .await
3396            }
3397        }
3398    }
3399
3400    fn log_supervisor_request_received(
3401        &self,
3402        ctx: &RouteCtx,
3403        corr: u64,
3404        op: &'static str,
3405        module_id: Option<&str>,
3406        enabled: Option<bool>,
3407    ) -> Result<(), RouterError> {
3408        let caller = self.supervisor_requester(ctx, corr)?;
3409        let caller = principal_label(&caller);
3410        // `Option` fields are recorded only when present, so a request with no
3411        // module id (rescan) or no enabled flag simply omits that field.
3412        info!(
3413            target: "control",
3414            op,
3415            module_id,
3416            enabled,
3417            connection_id = ctx.connection_id.get(),
3418            caller = %caller,
3419            "supervisor request received"
3420        );
3421        Ok(())
3422    }
3423
3424    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3425        Ok(self
3426            .registry
3427            .get_module_by_connection(ctx.connection_id)
3428            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3429            .map(|registration| Principal::Reserved {
3430                module_id: registration.manifest.module_id,
3431            })
3432            .unwrap_or(Principal::Direct))
3433    }
3434
3435    fn handle_module_control_request(
3436        &self,
3437        connection_id: ConnectionId,
3438        frame: Frame,
3439        request: ModuleControlRequestFromModule,
3440    ) -> Result<Vec<Frame>, RouterError> {
3441        match request {
3442            ModuleControlRequestFromModule::CatalogUpdate {
3443                provides,
3444                capabilities,
3445                ready,
3446            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3447            ModuleControlRequestFromModule::LiveRoots {} => {
3448                let registered = self
3449                    .registry
3450                    .get_module_by_connection(connection_id)
3451                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3452                let Some(registration) = registered else {
3453                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3454                };
3455                let response = self
3456                    .forwarding
3457                    .live_roots(&registration.manifest.module_id)
3458                    .map_err(RouterError::Forwarding)?;
3459                Ok(vec![control_response_body_frame(
3460                    &frame,
3461                    &response,
3462                    "ModuleControlResponseToModule::LiveRoots",
3463                )?])
3464            }
3465            ModuleControlRequestFromModule::AgentCommands {
3466                started,
3467                finished,
3468                known,
3469            } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3470            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3471                self.handle_scope_sync(connection_id, frame, generation, scopes)
3472            }
3473            ModuleControlRequestFromModule::ScopeApply {
3474                generation,
3475                upsert,
3476                end,
3477            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3478            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3479                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3480            }
3481            _ => Ok(vec![control_error_frame(
3482                &frame,
3483                "unsupported_control_frame",
3484                "unsupported module control operation",
3485            )?]),
3486        }
3487    }
3488
3489    fn handle_scope_served_by(
3490        &self,
3491        connection_id: ConnectionId,
3492        frame: Frame,
3493    ) -> Result<Vec<Frame>, RouterError> {
3494        let registered = self
3495            .registry
3496            .get_module_by_connection(connection_id)
3497            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3498        let Some(registered) = registered else {
3499            return Ok(vec![control_error_frame(
3500                &frame,
3501                "not_registered",
3502                "scope.served_by requires an active module registration owned by this connection",
3503            )?]);
3504        };
3505        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3506        {
3507            Ok(request) => request,
3508            Err(err) => {
3509                return Ok(vec![control_error_frame(
3510                    &frame,
3511                    "invalid_control_body",
3512                    format!("malformed scope.served_by body: {err}"),
3513                )?])
3514            }
3515        };
3516        let answer = self
3517            .scopes
3518            .read()
3519            .unwrap_or_else(|poisoned| poisoned.into_inner())
3520            .served_by(
3521                &registered.manifest.module_id,
3522                &request.scope_ref,
3523                request.scope_epoch,
3524                &request.module_id,
3525            );
3526        let reply = subc_control::ScopeServedByReply::new(
3527            answer,
3528            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3529        );
3530        Ok(vec![control_response_body_frame(
3531            &frame,
3532            &reply,
3533            "ScopeServedByReply",
3534        )?])
3535    }
3536
3537    fn handle_operator_confirm(
3538        &self,
3539        ctx: &RouteCtx,
3540        frame: Frame,
3541    ) -> Result<Vec<Frame>, RouterError> {
3542        use crate::operator_confirm::{audit, Outcome};
3543        let registration = self
3544            .registry
3545            .get_module_by_connection(ctx.connection_id)
3546            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3547        let Some(registration) = registration else {
3548            let outcome = Outcome::refusal("not_registered");
3549            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3550            return Ok(vec![outcome.frame(&frame)]);
3551        };
3552        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3553            Ok(request) => request,
3554            Err(_) => {
3555                let outcome = Outcome::refusal("invalid_control_body");
3556                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3557                return Ok(vec![outcome.frame(&frame)]);
3558            }
3559        };
3560        let declared = matches!(
3561            request.kind,
3562            Some(subc_protocol::session::OperatorConfirmKind::Operation)
3563        ) && request.operation.as_ref().is_some_and(|operation| {
3564            registration
3565                .manifest
3566                .confirm_operations
3567                .as_ref()
3568                .is_some_and(|operations| operations.contains(operation))
3569        });
3570        let module_id = registration.manifest.module_id;
3571        // Read the launch nonce (under its own lock) before taking the forwarding
3572        // table's lock below: holding forwarding while waiting on another daemon
3573        // lock risks a lock-order deadlock with paths that take them the other way.
3574        let nonce = self
3575            .hello_launch_nonces
3576            .lock()
3577            .unwrap_or_else(|p| p.into_inner())
3578            .nonce(ctx.connection_id)
3579            .map(str::to_owned);
3580        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3581            self.supervisor
3582                .spawned_consumer_authorized(&module_id, nonce)
3583        });
3584        let confirms = self.forwarding.operator_confirms();
3585        self.forwarding
3586            .with_operator_route(
3587                ctx.connection_id,
3588                request.route_channel,
3589                request.route_epoch,
3590                |binding| {
3591                    let shortcut = binding
3592                        .filter(|binding| {
3593                            nonce_proven
3594                                && crate::operator_confirm::valid_summary(&request.summary)
3595                                && declared
3596                                && crate::operator_confirm::ShortcutContext::route_eligible(
3597                                    &self.operator_app_policy,
3598                                    &self.launch_table,
3599                                    binding,
3600                                )
3601                        })
3602                        .map(|binding| {
3603                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3604                                &self.operator_app_policy,
3605                                binding,
3606                            );
3607                            let stamp = if phone.is_some() {
3608                                None
3609                            } else {
3610                                binding.operator_app.clone()
3611                            };
3612                            self.shortcut_context(
3613                                stamp,
3614                                phone,
3615                                Some((request.route_channel, request.route_epoch)),
3616                            )
3617                        });
3618                    confirms.admit_with_shortcut(
3619                        ctx,
3620                        frame,
3621                        module_id,
3622                        nonce_proven,
3623                        request,
3624                        binding,
3625                        shortcut,
3626                    )
3627                },
3628            )
3629            .map_err(RouterError::Forwarding)
3630    }
3631
3632    fn shortcut_context(
3633        &self,
3634        stamp: Option<crate::operator_apps::OperatorAppStamp>,
3635        phone_fingerprint: Option<String>,
3636        route: Option<(u16, u32)>,
3637    ) -> crate::operator_confirm::ShortcutContext {
3638        crate::operator_confirm::ShortcutContext {
3639            ledger: self.activity.clone(),
3640            forwarding: self.forwarding.clone(),
3641            launches: self.launch_table.clone(),
3642            stamp,
3643            phone_fingerprint,
3644            route,
3645            connections: self.operator_apps.clone(),
3646            #[cfg(test)]
3647            hooks: Default::default(),
3648        }
3649    }
3650
3651    /// Explicit test-only phone policy; daemon configuration cannot select it.
3652    #[cfg(any(test, feature = "test-support"))]
3653    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3654        self.operator_app_policy.phone_enabled = enabled;
3655        self
3656    }
3657
3658    /// Resolve folders before taking the scope-table lock: a slow filesystem
3659    /// must not hold up admissions for unrelated scoped routes.
3660    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3661        let mut scopes = ScopeRecords::from(scopes);
3662        for (index, record) in scopes.records.iter_mut().enumerate() {
3663            if let Some(refusal) = Self::canonicalize_workdir(record) {
3664                scopes.refusals.insert(index, refusal);
3665            }
3666        }
3667        scopes
3668    }
3669
3670    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3671        let workdir = record.workdir.as_mut()?;
3672        let path = std::path::Path::new(workdir);
3673        if !path.is_absolute() {
3674            return Some((
3675                error_codes::INVALID_WORKDIR,
3676                "workdir must be absolute".to_string(),
3677            ));
3678        }
3679        match ProjectRootId::from_path_allowing_missing(path) {
3680            Ok(root) => match root.as_path().to_str() {
3681                Some(canonical) => {
3682                    *workdir = canonical.to_string();
3683                    None
3684                }
3685                None => Some((
3686                    error_codes::INVALID_WORKDIR,
3687                    "canonical workdir is not UTF-8".to_string(),
3688                )),
3689            },
3690            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3691        }
3692    }
3693
3694    /// `scope.sync`: the owner is the module registered on this connection.
3695    /// A connection with no registration (every client connection, `direct`
3696    /// included) is refused `not_registered` before the table is consulted.
3697    fn handle_scope_sync(
3698        &self,
3699        connection_id: ConnectionId,
3700        frame: Frame,
3701        generation: u64,
3702        scopes: Vec<ScopeRecord>,
3703    ) -> Result<Vec<Frame>, RouterError> {
3704        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3705    }
3706
3707    fn handle_scope_change(
3708        &self,
3709        connection_id: ConnectionId,
3710        frame: Frame,
3711        generation: u64,
3712        scopes: Vec<ScopeRecord>,
3713        end: Option<Vec<ScopeEnd>>,
3714    ) -> Result<Vec<Frame>, RouterError> {
3715        let is_apply = end.is_some();
3716        let Some(registration) = self
3717            .registry
3718            .get_module_by_connection(connection_id)
3719            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3720        else {
3721            return Ok(vec![control_error_frame(
3722                &frame,
3723                "not_registered",
3724                "scope.sync and scope.apply require an active module registration owned by this connection",
3725            )?]);
3726        };
3727        let owner = registration.manifest.module_id;
3728        let scopes = Self::prepare_scope_records(scopes);
3729        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3730        let is_current_launch = |connection: ConnectionId| {
3731            self.hello_launch_nonces
3732                .lock()
3733                .unwrap_or_else(|poisoned| poisoned.into_inner())
3734                .presented(connection, current_nonce.as_deref())
3735        };
3736        // Lock order is the scope table, then the forwarding table: the new
3737        // tags are published, and the routes the change closes are selected,
3738        // while the scope table is still write-locked, so no admission can read
3739        // a record whose tag is not yet published.
3740        let mut table = self
3741            .scopes
3742            .write()
3743            .unwrap_or_else(|poisoned| poisoned.into_inner());
3744        let outcome = match end {
3745            Some(end) => table.apply(
3746                &owner,
3747                connection_id,
3748                is_current_launch,
3749                generation,
3750                scopes,
3751                end,
3752            ),
3753            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3754        };
3755        let drained = match &outcome {
3756            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3757            Err(_) => Vec::new(),
3758        };
3759        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3760        drop(table);
3761        if has_deadlines {
3762            self.scope_deadline_added.notify_one();
3763        }
3764        match outcome {
3765            Ok(applied) => {
3766                let counts = ScopeOutcomeCounts::of(&applied.results);
3767                info!(
3768                    owner = %owner,
3769                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3770                    generation,
3771                    records = applied.results.len(),
3772                    created = counts.created,
3773                    replaced = counts.replaced,
3774                    updated = counts.updated,
3775                    unchanged = counts.unchanged,
3776                    refused = counts.refused,
3777                    ended = applied.ended.len(),
3778                    tag_changes = applied.tag_changes.len(),
3779                    routes_closed = drained.len(),
3780                    "scope change accepted"
3781                );
3782                // An accepted scope change can still refuse individual records, and the
3783                // owner is the only party that sees the reply. Name them here so
3784                // an operator can tell a refused session from a missing one
3785                // without the owner's logs. Capped so a call that refuses
3786                // thousands cannot flood the log; the count above is complete.
3787                for refused in applied
3788                    .results
3789                    .iter()
3790                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3791                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3792                {
3793                    warn!(
3794                        owner = %owner,
3795                        generation,
3796                        scope_ref = %refused.scope_ref,
3797                        scope_epoch = refused.scope_epoch,
3798                        code = refused.code.as_deref().unwrap_or(""),
3799                        "scope record refused"
3800                    );
3801                }
3802                self.close_scope_drained_routes(drained);
3803                let response = if is_apply {
3804                    ModuleControlResponseToModule::ScopeApply {
3805                        generation,
3806                        results: applied.results,
3807                        end_results: applied.end_results,
3808                        ended: applied.ended,
3809                    }
3810                } else {
3811                    ModuleControlResponseToModule::ScopeSync {
3812                        generation,
3813                        results: applied.results,
3814                        ended: applied.ended,
3815                    }
3816                };
3817                Ok(vec![control_response_body_frame(
3818                    &frame,
3819                    &response,
3820                    "scope change response",
3821                )?])
3822            }
3823            Err(refusal) => {
3824                info!(
3825                    owner = %owner,
3826                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3827                    generation,
3828                    code = refusal.code,
3829                    "scope change refused"
3830                );
3831                Ok(vec![control_error_frame(
3832                    &frame,
3833                    refusal.code,
3834                    refusal.message,
3835                )?])
3836            }
3837        }
3838    }
3839
3840    /// Tell both ends of each route a scope change closed. The module gets a
3841    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3842    /// the client's route handle. The client also gets `route.closed` with the
3843    /// scope reason, one push per module and reason, so it can tell a revoked
3844    /// route from an ordinary close and not reopen it.
3845    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3846        if drained.is_empty() {
3847            return;
3848        }
3849        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3850            BTreeMap::new();
3851        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3852        for route in drained {
3853            warn!(
3854                module_id = %route.module_id,
3855                reason = ?route.reason,
3856                client_connection_id = route.client.connection_id.get(),
3857                route_channel = route.client.channel,
3858                "closing route because its scope changed"
3859            );
3860            pushes
3861                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3862                .or_insert_with(|| (route.reason, Vec::new()))
3863                .1
3864                .push(EndpointRoute {
3865                    goodbye_target: route.client.clone(),
3866                    principal: Principal::Unverified,
3867                    bound_at: Instant::now(),
3868                    draining: false,
3869                    drain_reason: None,
3870                });
3871            goodbyes.push(route.module);
3872            goodbyes.push(route.client);
3873        }
3874        for ((module_id, _), (reason, routes)) in pushes {
3875            send_route_control_pushes(
3876                &self.forwarding,
3877                routes,
3878                ClientControlPush::RouteClosed {
3879                    module_id,
3880                    channels: Vec::new(),
3881                    reason,
3882                    drained: false,
3883                    abandoned: 0,
3884                    excluded_subscriptions: 0,
3885                    terminal: Some(false),
3886                },
3887            );
3888        }
3889        self.emit_route_goodbyes(goodbyes);
3890    }
3891
3892    /// `scope.describe`: any registered module may read any scope, because a
3893    /// provider must read the scope a route it serves is stamped with.
3894    fn handle_scope_describe(
3895        &self,
3896        connection_id: ConnectionId,
3897        frame: Frame,
3898        owner: Principal,
3899        scope_ref: String,
3900    ) -> Result<Vec<Frame>, RouterError> {
3901        let registered = self
3902            .registry
3903            .get_module_by_connection(connection_id)
3904            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3905        let Some(registered) = registered else {
3906            return Ok(vec![control_error_frame(
3907                &frame,
3908                "not_registered",
3909                "scope.describe requires an active module registration owned by this connection",
3910            )?]);
3911        };
3912        let scopes = self
3913            .scopes
3914            .read()
3915            .unwrap_or_else(|poisoned| poisoned.into_inner());
3916        if scopes.is_federated_owner(&owner)
3917            && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
3918        {
3919            return Ok(vec![control_error_frame(
3920                &frame,
3921                error_codes::SCOPE_FEDERATED_UNSUPPORTED,
3922                "scope.describe of the federated owner requires scope-federated/v1",
3923            )?]);
3924        }
3925        let mut description = scopes.describe(&owner, &scope_ref);
3926        drop(scopes);
3927        // Describe readers also decode strict ScopeAttributes. Attribution is
3928        // safe to omit unless the requesting module declares it understands it.
3929        if !registered
3930            .manifest
3931            .capabilities
3932            .as_ref()
3933            .is_some_and(|caps| {
3934                caps.provides
3935                    .iter()
3936                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3937            })
3938        {
3939            if let Some(stamp) = description.stamp.as_mut() {
3940                stamp.attributes.on_behalf_of_agent = None;
3941            }
3942        }
3943        if !registered
3944            .manifest
3945            .capabilities
3946            .as_ref()
3947            .is_some_and(|caps| {
3948                caps.provides
3949                    .iter()
3950                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3951            })
3952        {
3953            if let Some(stamp) = description.stamp.as_mut() {
3954                stamp.attributes.flow_version = None;
3955                stamp.attributes.flow_code_hash = None;
3956                stamp.attributes.flow_install_id = None;
3957            }
3958        }
3959        let owner_configured = match &owner {
3960            // Ask whether the owner is configured (`is_configured`), not
3961            // whether it is on the roster (`get(..).is_some()`): a supervised
3962            // module's process can register and describe a scope before the
3963            // supervisor has put it on the roster.
3964            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3965            _ => false,
3966        };
3967        let response = ModuleControlResponseToModule::ScopeDescribe {
3968            status: description.status,
3969            scope_epoch: description.scope_epoch,
3970            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3971            owner_synced: description.owner_synced,
3972            owner_configured,
3973            scope: description.stamp,
3974        };
3975        Ok(vec![control_response_body_frame(
3976            &frame,
3977            &response,
3978            "ModuleControlResponseToModule::ScopeDescribe",
3979        )?])
3980    }
3981
3982    fn handle_catalog_update(
3983        &self,
3984        connection_id: ConnectionId,
3985        frame: Frame,
3986        provides: Vec<ProviderRole>,
3987        capabilities: Option<CapabilityDeclarations>,
3988        ready: Option<bool>,
3989    ) -> Result<Vec<Frame>, RouterError> {
3990        self.refresh_capability_requirements();
3991        let Some(registration) = self
3992            .registry
3993            .get_module_by_connection(connection_id)
3994            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3995        else {
3996            return Ok(vec![control_error_frame(
3997                &frame,
3998                "not_registered",
3999                "catalog.update requires an active module registration owned by this connection",
4000            )?]);
4001        };
4002
4003        if let Some(message) =
4004            catalog_update_frozen_field_message(&registration.manifest, &provides)
4005        {
4006            return Ok(vec![control_error_frame(
4007                &frame,
4008                "catalog_update_frozen_field",
4009                message,
4010            )?]);
4011        }
4012
4013        let mut candidate = registration.manifest.clone();
4014        candidate.provides = provides.clone();
4015        candidate.capabilities = capabilities
4016            .clone()
4017            .or_else(|| registration.manifest.capabilities.clone());
4018        if let Some(rich_ui) = &candidate.rich_ui {
4019            if let Err(err) = rich_ui.validate() {
4020                return Ok(vec![control_error_frame(
4021                    &frame,
4022                    "invalid_manifest",
4023                    err.to_string(),
4024                )?]);
4025            }
4026        }
4027        if let Err(err) = candidate.validate_capability_grammar() {
4028            return Ok(vec![control_error_frame(
4029                &frame,
4030                "invalid_capability_grammar",
4031                err.to_string(),
4032            )?]);
4033        }
4034
4035        // Updates must honor the same reserved owner as initial registration;
4036        // otherwise an empty HELLO could acquire the claim after admission.
4037        let mut conflicts = self
4038            .capability_evaluator
4039            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4040        if let Some(conflict) = conflicts.first() {
4041            let message = format!(
4042                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4043                conflict.capability, conflict.claimants[0], candidate.module_id
4044            );
4045            for conflict in &mut conflicts {
4046                conflict.source = DuplicateClaimSource::CatalogUpdate;
4047            }
4048            log_duplicate_claim_events(conflicts);
4049            return Ok(vec![control_error_frame(
4050                &frame,
4051                "reserved_capability",
4052                message,
4053            )?]);
4054        }
4055
4056        let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4057            Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4058                &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4059                "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4060            )?]),
4061            result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4062        };
4063        if updated.is_none() {
4064            return Ok(vec![control_error_frame(
4065                &frame,
4066                "not_registered",
4067                "catalog.update requires an active module registration owned by this connection",
4068            )?]);
4069        }
4070        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4071            log_duplicate_claim_events(
4072                self.capability_evaluator
4073                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
4074            );
4075        }
4076        if capability_census_trigger(
4077            registration.manifest.capabilities.as_ref(),
4078            updated
4079                .as_ref()
4080                .and_then(|entry| entry.manifest.capabilities.as_ref()),
4081        ) {
4082            self.enforce_capability_denies();
4083        }
4084        self.refresh_capability_requirements();
4085
4086        let response = ModuleControlResponseToModule::CatalogUpdate {};
4087        control_response_body_frame(
4088            &frame,
4089            &response,
4090            "ModuleControlResponseToModule::CatalogUpdate",
4091        )
4092        .map(|frame| vec![frame])
4093    }
4094
4095    pub(crate) fn with_activity_ledger_config(
4096        mut self,
4097        run_dir: &Path,
4098        incarnation: String,
4099        config: crate::activity::ActivityConfig,
4100    ) -> Self {
4101        self.activity = Some(crate::activity::ActivityLedger::open_config(
4102            run_dir.join("activity.jsonl"),
4103            incarnation,
4104            config,
4105        ));
4106        self.supervisor.set_activity(self.activity.clone());
4107        self
4108    }
4109
4110    pub(crate) async fn configure_activity(
4111        &self,
4112        config: crate::activity::ActivityConfig,
4113    ) -> Result<(), &'static str> {
4114        match &self.activity {
4115            Some(ledger) => ledger.configure(config).await,
4116            None => Ok(()),
4117        }
4118    }
4119
4120    fn cancel_activity_subscription(
4121        &self,
4122        ctx: &RouteCtx,
4123        frame: &Frame,
4124    ) -> Result<bool, RouterError> {
4125        let task = self
4126            .activity_streams
4127            .lock()
4128            .unwrap_or_else(|p| p.into_inner())
4129            .remove(&(ctx.connection_id, frame.header.corr));
4130        let Some(task) = task else {
4131            return Ok(false);
4132        };
4133        task.abort();
4134        let end = Frame::build_with_version(
4135            response_version(frame),
4136            FrameType::StreamEnd,
4137            control_flags(),
4138            0,
4139            0,
4140            frame.header.corr,
4141            Vec::new(),
4142        )
4143        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4144        let sink = ctx.egress.clone();
4145        tokio::spawn(async move {
4146            let _ = sink.send(end).await;
4147        });
4148        Ok(true)
4149    }
4150
4151    async fn handle_activity_read(
4152        &self,
4153        ctx: &RouteCtx,
4154        frame: Frame,
4155    ) -> Result<Vec<Frame>, RouterError> {
4156        use subc_control::{
4157            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4158            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4159        };
4160        #[derive(Deserialize)]
4161        struct Identity {
4162            #[serde(default)]
4163            consumer_identity: Option<ConsumerIdentity>,
4164        }
4165        // A registered module connection already identifies its caller. Ignore
4166        // consumer_identity there so a module cannot borrow a mirror's identity.
4167        // Only separate consumer connections use the Direct/nonce trust boundary.
4168        let principal = match self
4169            .forwarding
4170            .module_id_for_connection(ctx.connection_id)
4171            .map_err(RouterError::Forwarding)?
4172        {
4173            Some(module_id) => Ok(Principal::Reserved { module_id }),
4174            None => {
4175                let identity: Identity = match serde_json::from_slice(&frame.body) {
4176                    Ok(identity) => identity,
4177                    Err(_) => {
4178                        return Ok(vec![control_error_frame(
4179                            &frame,
4180                            ACTIVITY_MALFORMED,
4181                            "malformed activity read",
4182                        )?])
4183                    }
4184                };
4185                self.route_open_principal(&frame, identity.consumer_identity)?
4186            }
4187        };
4188        let admitted = matches!(principal, Ok(Principal::Direct))
4189            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4190        if !admitted {
4191            return Ok(vec![control_error_frame(
4192                &frame,
4193                ACTIVITY_READ_FORBIDDEN,
4194                "activity reads require an operator or trusted mirror principal",
4195            )?]);
4196        }
4197        let Some(ledger) = &self.activity else {
4198            return Ok(vec![control_error_frame(
4199                &frame,
4200                ACTIVITY_UNAVAILABLE,
4201                "activity ledger unavailable",
4202            )?]);
4203        };
4204        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4205            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4206        {
4207            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4208                Ok(request) => ledger.query(request).await,
4209                Err(_) => Err(ACTIVITY_MALFORMED),
4210            };
4211            return match result {
4212                Ok(reply) => Ok(vec![control_response_body_frame(
4213                    &frame,
4214                    &reply,
4215                    ops::ACTIVITY_QUERY,
4216                )?]),
4217                Err(code) => Ok(vec![control_error_frame(
4218                    &frame,
4219                    code,
4220                    "activity query refused",
4221                )?]),
4222            };
4223        }
4224        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4225            Ok(request) => request,
4226            Err(_) => {
4227                return Ok(vec![control_error_frame(
4228                    &frame,
4229                    ACTIVITY_MALFORMED,
4230                    "malformed activity subscription",
4231                )?])
4232            }
4233        };
4234        let mut subscription = match ledger.subscribe(request.since).await {
4235            Ok(subscription) => subscription,
4236            Err(code) => {
4237                return Ok(vec![control_error_frame(
4238                    &frame,
4239                    code,
4240                    "activity subscription refused",
4241                )?])
4242            }
4243        };
4244        let key = (ctx.connection_id, frame.header.corr);
4245        let sink = ctx.egress.clone();
4246        let mut streams = self
4247            .activity_streams
4248            .lock()
4249            .unwrap_or_else(|p| p.into_inner());
4250        streams.retain(|_, task| !task.is_finished());
4251        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4252            return Ok(vec![control_error_frame(
4253                &frame,
4254                "duplicate_subscription",
4255                "activity subscription correlation id already in use",
4256            )?]);
4257        }
4258        let task = tokio::spawn(async move {
4259            loop {
4260                let entry = if let Some(entry) = subscription.replay.pop_front() {
4261                    Some(entry)
4262                } else {
4263                    subscription.receiver.recv().await
4264                };
4265                let Some(entry) = entry else {
4266                    break;
4267                };
4268                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4269                    return;
4270                };
4271                let Ok(next) = Frame::build_with_version(
4272                    response_version(&frame),
4273                    FrameType::StreamData,
4274                    control_flags(),
4275                    0,
4276                    0,
4277                    frame.header.corr,
4278                    body,
4279                ) else {
4280                    return;
4281                };
4282                if sink.send(next).await.is_err() {
4283                    return;
4284                }
4285            }
4286            // Dropping a full subscriber's sender drains its accepted entries
4287            // before the terminal error, preserving their order on the wire.
4288            if let Some(code) = subscription.terminal_code() {
4289                if let Ok(end) =
4290                    control_error_frame(&frame, code, "activity subscriber fell behind")
4291                {
4292                    let _ = sink.send(end).await;
4293                }
4294            }
4295        });
4296        streams.insert(key, task.abort_handle());
4297        Ok(Vec::new())
4298    }
4299
4300    async fn handle_activity_record(
4301        &self,
4302        ctx: &RouteCtx,
4303        frame: Frame,
4304    ) -> Result<Vec<Frame>, RouterError> {
4305        use subc_control::{
4306            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4307            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4308        };
4309        let result = async {
4310            let module = self
4311                .forwarding
4312                .module_id_for_connection(ctx.connection_id)
4313                .ok()
4314                .flatten()
4315                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4316            let request: ActivityRecordRequest =
4317                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4318            crate::activity::validate(&module, &request)?;
4319            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4320            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4321            let at_ms = std::time::SystemTime::now()
4322                .duration_since(std::time::UNIX_EPOCH)
4323                .unwrap_or_default()
4324                .as_millis() as u64;
4325            ledger.record(module, request, actor, at_ms).await
4326        }
4327        .await;
4328        match result {
4329            Ok(entry) => Ok(vec![control_response_body_frame(
4330                &frame,
4331                &ActivityRecordResponse { seq: entry.seq },
4332                "activity.record",
4333            )?]),
4334            Err(code) => Ok(vec![control_error_frame(
4335                &frame,
4336                code,
4337                "activity recording refused",
4338            )?]),
4339        }
4340    }
4341
4342    fn activity_actor(
4343        &self,
4344        connection: ConnectionId,
4345        module: &str,
4346        on: Option<&subc_control::ActivityOn>,
4347    ) -> Result<subc_control::ActivityActor, &'static str> {
4348        use crate::forwarding::{DataRoute, DataRouteState};
4349        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4350        let principal = |id: &str| Principal::Reserved {
4351            module_id: id.to_owned(),
4352        };
4353        let mut actor = ActivityActor {
4354            principal: principal(module),
4355            scope_owner: None,
4356            agent_id: None,
4357            run_id: None,
4358            flow_id: None,
4359            project_root: None,
4360            workdir: None,
4361            project_id: None,
4362        };
4363        let scope = match on {
4364            None => return Ok(actor),
4365            Some(ActivityOn::Scope(scope)) => {
4366                let selector = subc_protocol::scope::ScopeSelector::new(
4367                    principal(module),
4368                    scope.scope_ref.clone(),
4369                    Some(scope.epoch),
4370                );
4371                self.scopes
4372                    .read()
4373                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4374                    .attribute(&principal(module), module, &selector, true)
4375                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4376            }
4377            Some(ActivityOn::Route(route)) => {
4378                let DataRoute::Module(DataRouteState::Bound(binding)) = self
4379                    .forwarding
4380                    .lookup_data_route(connection, route.channel, route.epoch)
4381                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4382                else {
4383                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4384                };
4385                if binding.module_id != module {
4386                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4387                }
4388                actor.principal = binding.principal.clone();
4389                actor.project_root = binding
4390                    .project_root
4391                    .as_ref()
4392                    .map(|root| root.as_path().to_path_buf());
4393                let Some(bound) = &binding.scope else {
4394                    return Ok(actor);
4395                };
4396                let owner = principal(&bound.owner);
4397                let selector = subc_protocol::scope::ScopeSelector::new(
4398                    owner.clone(),
4399                    bound.scope_ref.clone(),
4400                    Some(bound.tag.scope_epoch),
4401                );
4402                let admitted = self
4403                    .scopes
4404                    .read()
4405                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4406                    .attribute(&owner, module, &selector, true)
4407                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4408                if admitted.tag != bound.tag {
4409                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4410                }
4411                admitted
4412            }
4413        };
4414        actor.scope_owner = Some(scope.stamp.owner);
4415        actor.agent_id = scope.stamp.attributes.agent_id;
4416        actor.run_id = scope.stamp.attributes.run_id;
4417        actor.flow_id = scope.stamp.attributes.flow_id;
4418        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4419        actor.project_id = scope.stamp.project_id;
4420        Ok(actor)
4421    }
4422
4423    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4424        self.refresh_capability_requirements();
4425        // A bare connection count is ambiguous between many clients holding a
4426        // route each and one client accumulating hundreds, so publish the
4427        // concentration alongside it. Route state is best-effort here: a
4428        // diagnostic endpoint must still answer if the forwarding lock is
4429        // contended.
4430        let mut counters = self.counters.snapshot();
4431        let losses = self.resource_tracker.loss_diagnostics();
4432        if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4433            if let Some(obj) = counters.as_object_mut() {
4434                obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4435            }
4436        }
4437        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4438            obj.insert(
4439                "activity_unparseable_lines".into(),
4440                ledger.read_errors().into(),
4441            );
4442            obj.insert(
4443                "activity_lifecycle_append_failures".into(),
4444                ledger.lifecycle_failures().into(),
4445            );
4446            obj.insert(
4447                "activity_started_digest_skips".into(),
4448                ledger.lifecycle_digest_skips().into(),
4449            );
4450        }
4451        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4452            self.forwarding.client_route_concentration(),
4453            counters.as_object_mut(),
4454        ) {
4455            obj.insert(
4456                "client_connections_with_routes".into(),
4457                connections_with_routes.into(),
4458            );
4459            obj.insert("max_routes_on_one_connection".into(), max.into());
4460        }
4461        // A module that is being fast-refused and a module that is fine look
4462        // identical from a client that retries and succeeds, so name the open
4463        // breakers here. This rides the existing free-form counters object
4464        // rather than a new wire field, so no sibling that deserializes
4465        // `ServerDescribe` has to be rebuilt to keep reading it.
4466        if let (Some(open_breakers), Some(obj)) = (
4467            self.route_bind_breakers.open_snapshot(),
4468            counters.as_object_mut(),
4469        ) {
4470            obj.insert("route_bind_breakers_open".into(), open_breakers);
4471        }
4472        // A daemon whose local endpoint could not be bound keeps serving TCP,
4473        // and the connection file names no endpoint. Name the reason here, in
4474        // the free-form counters object, so an operator reading daemon facts
4475        // sees it without searching the log, and no wire field changes.
4476        if let (Some(reason), Some(obj)) = (
4477            self.local_endpoint_unavailable.clone(),
4478            counters.as_object_mut(),
4479        ) {
4480            obj.insert("local_endpoint_unavailable".into(), reason.into());
4481        }
4482        let response = ClientControlResponse::ServerDescribe {
4483            protocol_ver: PROTOCOL_VERSION,
4484            subc_ops: subc_ops(),
4485            capabilities: self.describe_capabilities(),
4486            connected_clients: self.connected_clients.count(),
4487            counters: Some(counters),
4488            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4489            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4490            capability_requirements: self.capability_requirement_statuses(),
4491            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4492            config_path: self
4493                .rescan
4494                .as_ref()
4495                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4496        };
4497        Ok(vec![control_response_body_frame(
4498            &frame,
4499            &response,
4500            "ClientControlResponse::ServerDescribe",
4501        )?])
4502    }
4503
4504    fn handle_catalog_list(
4505        &self,
4506        frame: Frame,
4507        module_id: Option<String>,
4508    ) -> Result<Vec<Frame>, RouterError> {
4509        let (generation, modules) = self
4510            .registry
4511            .list_modules_with_nonce_verification()
4512            .map_err(|err| {
4513                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4514            })?;
4515        let entries = modules
4516            .into_iter()
4517            .filter(|(registration, _)| {
4518                module_id
4519                    .as_deref()
4520                    .map(|wanted| registration.manifest.module_id == wanted)
4521                    .unwrap_or(true)
4522            })
4523            .map(|(registration, nonce_verified)| {
4524                let not_ready = self.not_ready_reason(&registration);
4525                let roles = registration.manifest.provides;
4526                CatalogEntry::new(
4527                    registration.manifest.module_id,
4528                    roles,
4529                    registration.control_ops,
4530                )
4531                .with_ready(not_ready.is_none())
4532                .with_display_name(registration.manifest.display_name)
4533                .with_not_ready(not_ready)
4534                .with_module_version(Some(registration.manifest.module_version))
4535                .with_provenance(registration.manifest.provenance)
4536                .with_capabilities(registration.manifest.capabilities)
4537                .with_self_signals(registration.manifest.self_signals)
4538                .with_events(registration.manifest.events)
4539                .with_operation_delivery(registration.manifest.operation_delivery)
4540                .with_operation_payload(registration.manifest.operation_payload)
4541                .with_flow_operations(registration.manifest.flow_operations)
4542                .with_confirm_operations(registration.manifest.confirm_operations)
4543                .with_rich_ui(registration.manifest.rich_ui)
4544                .with_role_declarations(registration.manifest.role_declarations)
4545                .with_nonce_verified(nonce_verified)
4546            })
4547            .collect();
4548        let response = ClientControlResponse::CatalogList {
4549            generation,
4550            modules: entries,
4551            subc_ops: subc_ops(),
4552        };
4553        Ok(vec![control_response_body_frame(
4554            &frame,
4555            &response,
4556            "ClientControlResponse::CatalogList",
4557        )?])
4558    }
4559
4560    /// The principal a route.open's `consumer_identity` attests: `Direct`
4561    /// when it presents none, `Reserved` when its launch nonce verifies, and
4562    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4563    /// can run before any refusal check without changing which check refuses.
4564    fn attest_route_open_consumer(
4565        &self,
4566        consumer_identity: Option<&ConsumerIdentity>,
4567    ) -> Option<Principal> {
4568        let Some(consumer_identity) = consumer_identity else {
4569            return Some(Principal::Direct);
4570        };
4571        self.supervisor
4572            .spawned_consumer_authorized(
4573                &consumer_identity.module_id,
4574                &consumer_identity.launch_nonce,
4575            )
4576            .then(|| Principal::Reserved {
4577                module_id: consumer_identity.module_id.clone(),
4578            })
4579    }
4580
4581    /// The caller fields for a refusal that only has the raw request frame,
4582    /// such as the connection-level admission refusal sent before the request
4583    /// reaches [`Self::handle_route_open`].
4584    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4585        let Ok(ClientControlRequest::RouteOpen {
4586            identity,
4587            consumer_identity,
4588            scope,
4589            ..
4590        }) = parse_client_control_request(&frame.body)
4591        else {
4592            return RouteOpenCaller::unknown();
4593        };
4594        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4595        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4596    }
4597
4598    fn route_open_principal(
4599        &self,
4600        frame: &Frame,
4601        consumer_identity: Option<ConsumerIdentity>,
4602    ) -> Result<Result<Principal, Frame>, RouterError> {
4603        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4604        self.route_open_principal_attested(frame, consumer_identity, attested)
4605    }
4606
4607    /// [`Self::route_open_principal`] with `attested` already taken by
4608    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4609    /// open checks the launch nonce once.
4610    fn route_open_principal_attested(
4611        &self,
4612        frame: &Frame,
4613        consumer_identity: Option<ConsumerIdentity>,
4614        attested: Option<Principal>,
4615    ) -> Result<Result<Principal, Frame>, RouterError> {
4616        if let Some(principal) = attested {
4617            return Ok(Ok(principal));
4618        }
4619        let Some(consumer_identity) = consumer_identity else {
4620            return Ok(Ok(Principal::Direct));
4621        };
4622
4623        Ok(Err(control_error_frame(
4624            frame,
4625            "bad_consumer_identity",
4626            format!(
4627                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4628                consumer_identity.module_id
4629            ),
4630        )?))
4631    }
4632
4633    /// Ordinary `route.open` refusals go through here; admission and breaker
4634    /// refusals log separately with their capacity or breaker state. The daemon can
4635    /// attest which code it sent: without the event, a client's "the daemon
4636    /// refused me" and the daemon's own view could only be reconciled by
4637    /// argument. Malformed input (`invalid_project_root`) does not come here;
4638    /// rejecting a request that was never a valid open is not a refusal of one.
4639    #[allow(clippy::too_many_arguments)]
4640    fn route_open_refusal_frame(
4641        &self,
4642        ctx: &RouteCtx,
4643        caller: &RouteOpenCaller,
4644        frame: &Frame,
4645        module_id: &str,
4646        reason: &'static str,
4647        code: &'static str,
4648        message: impl Into<String>,
4649    ) -> Result<Frame, RouterError> {
4650        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4651        control_error_frame(frame, code, message.into())
4652    }
4653
4654    /// Answer a waiting `route.open` whose relayed bind was refused.
4655    ///
4656    /// `daemon_authored` is provenance, not a code match: the daemon
4657    /// synthesized the refusal itself (a drain, a late reply, a scope that
4658    /// ended mid-bind) or the module refused the bind with its own body. A
4659    /// daemon refusal carries the daemon-origin bit; a module's own
4660    /// rejection is relayed with the bit clear, so a module that refuses
4661    /// with `module_reloading` still cannot mint a daemon refusal.
4662    #[allow(clippy::too_many_arguments)]
4663    fn route_bind_rejection(
4664        &self,
4665        ctx: &RouteCtx,
4666        caller: &RouteOpenCaller,
4667        frame: &Frame,
4668        target_module_id: &str,
4669        body: ErrorBody,
4670        daemon_authored: bool,
4671    ) -> Result<Vec<Frame>, RouterError> {
4672        // The daemon's own commit re-check refused the bind because the
4673        // scope ended or changed after admission. The module accepted;
4674        // counting it as a module rejection would blame the module. This
4675        // branch only runs for daemon-synthesized bodies: a module that
4676        // refuses with a scope code is answered as a module rejection below.
4677        if daemon_authored {
4678            let scope_code = match body.code.as_str() {
4679                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4680                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4681                _ => None,
4682            };
4683            if let Some(code) = scope_code {
4684                self.observe_route_open_refusal(
4685                    ctx,
4686                    caller,
4687                    target_module_id,
4688                    "scope_changed_before_commit",
4689                    code,
4690                );
4691                return Ok(vec![control_error_body_frame(frame, body)?]);
4692            }
4693        }
4694        self.counters
4695            .increment_route_open_refused("module_rejected");
4696        info!(
4697            target: "control",
4698            code = "module_rejected",
4699            module_code = ?body.code,
4700            module_id = ?target_module_id,
4701            connection_id = ctx.connection_id.get(),
4702            principal = %caller.principal,
4703            scope = %caller.scope,
4704            harness = %caller.harness,
4705            "route.open refused"
4706        );
4707        let refusal = if daemon_authored {
4708            control_error_body_frame(frame, body)?
4709        } else {
4710            control_relayed_error_body_frame(frame, body)?
4711        };
4712        Ok(vec![refusal])
4713    }
4714
4715    /// Refuse a `route.open` because the target module's bind-relay breaker is
4716    /// open, without attempting the relay.
4717    ///
4718    /// The wire code is `module_timeout`, which is the truth (the module has
4719    /// not been answering binds) and which both SDKs already classify as
4720    /// retryable with capped backoff. Reusing it is what keeps this change out
4721    /// of both SDKs; the daemon-side distinction lives in the counter key
4722    /// instead.
4723    ///
4724    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4725    /// While a breaker is open this fires on every open to that module, and the
4726    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4727    /// produced 261 lines about a single module inside 3000 lines of daemon
4728    /// log. The rare transitions are logged at warn/info instead and the volume
4729    /// is carried by the counter, so the evidence survives without the flood.
4730    /// The debug line keeps a per-refusal record reachable for whoever turns
4731    /// the level up.
4732    #[allow(clippy::too_many_arguments)]
4733    fn route_open_breaker_refusal_frame(
4734        &self,
4735        ctx: &RouteCtx,
4736        caller: &RouteOpenCaller,
4737        frame: &Frame,
4738        module_id: &str,
4739        consecutive_timeouts: u32,
4740        retry_in: Duration,
4741        probe_in_flight: bool,
4742    ) -> Result<Frame, RouterError> {
4743        self.counters
4744            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4745        debug!(
4746            target: "control",
4747            code = "module_timeout",
4748            module_id = ?module_id,
4749            connection_id = ctx.connection_id.get(),
4750            principal = %caller.principal,
4751            scope = %caller.scope,
4752            harness = %caller.harness,
4753            consecutive_timeouts,
4754            retry_in_ms = retry_in.as_millis() as u64,
4755            probe_in_flight,
4756            "route.open refused by open bind-relay breaker"
4757        );
4758        // Say what a caller can act on. An open bind-relay breaker means the
4759        // module timed out accepting several new routes in a row. The module
4760        // is still running and its established routes keep working; only new
4761        // route.open requests are refused until the cooldown ends and one
4762        // test route (the probe) gets through. A message that only counts
4763        // failed relays reads as "the module is down" to a worker that sees it.
4764        let detail = if probe_in_flight {
4765            "one test route is already being tried; retry once it settles".to_string()
4766        } else {
4767            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4768        };
4769        control_error_frame(
4770            frame,
4771            "module_timeout",
4772            format!(
4773                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4774                 timed out in a row); {detail}; its established routes are unaffected"
4775            ),
4776        )
4777    }
4778
4779    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4780    /// requester's bytes (an unknown target is whatever the client sent) and
4781    /// is Debug-formatted so control characters land in the log escaped
4782    /// rather than as terminal sequences for whoever tails it.
4783    ///
4784    /// `reason` names the check that refused, because one wire code has
4785    /// several senders: after a module registers, `target_unavailable` can
4786    /// come from a missing role, an inactive registration, a supervisor that
4787    /// has not marked the process live, a missing forwarding connection, or a
4788    /// failed relay, and a log that records only the code cannot say which of
4789    /// them fired. It is a static, daemon-chosen label per branch, so it is
4790    /// safe to print plainly and stays a closed set.
4791    fn observe_route_open_refusal(
4792        &self,
4793        ctx: &RouteCtx,
4794        caller: &RouteOpenCaller,
4795        module_id: &str,
4796        reason: &'static str,
4797        code: &'static str,
4798    ) {
4799        self.counters.increment_route_open_refused(code);
4800        info!(
4801            target: "control",
4802            code,
4803            reason,
4804            module_id = ?module_id,
4805            connection_id = ctx.connection_id.get(),
4806            principal = %caller.principal,
4807            scope = %caller.scope,
4808            harness = %caller.harness,
4809            "route.open refused"
4810        );
4811        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4812            self.route_outages.record_not_serving(module_id, reason);
4813        }
4814    }
4815
4816    /// Record an ACCEPTED route.open.
4817    ///
4818    /// Refusals have been logged and counted since the attestation work; accepts
4819    /// were invisible, so the daemon knew every principal it stamped and wrote
4820    /// none of them down. The party that attests the identity was the only party
4821    /// not recording it, which left a credential vault unable to name the sender
4822    /// of a call that reached it (claustrum #43) and left the launch-nonce
4823    /// concurrency question unanswerable from the outside.
4824    ///
4825    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4826    /// `code`/`module_id`/`connection_id` returns both directions of the same
4827    /// decision rather than two shapes a reader has to join by hand.
4828    ///
4829    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4830    /// and the difference carries information rather than being an
4831    /// inconsistency. This line is only reachable after a successful bind to a
4832    /// REGISTERED module, so the value has already passed HELLO validation
4833    /// including the path-hazard refusal and cannot contain control bytes. A
4834    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4835    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4836    ///
4837    /// Bare is also what every other daemon line already emits (`module
4838    /// registered`, `configured module supervised`). Shipping `?module_id` here
4839    /// made this instrument the only one in the file whose ids did not answer
4840    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4841    /// line whose whole purpose is being grepped beside its sibling.
4842    ///
4843    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4844    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4845    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4846    /// one are recorded identically. A test written against that harness passes
4847    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4848    /// green assertion that cannot fail. The same limit applies to the escaping
4849    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4850    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4851    /// Fencing either needs the real formatter, not the capture layer.
4852    ///
4853    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4854    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4855    /// to record. The ephemeral port would decay within minutes and answer only a
4856    /// live question. The identity question is instead answered by counting
4857    /// distinct live connections presenting one module's `consumer_identity` --
4858    /// "is anyone else holding this secret" rather than "is this the right
4859    /// process".
4860    fn observe_route_open_accept(
4861        &self,
4862        ctx: &RouteCtx,
4863        module_id: &str,
4864        principal: &str,
4865        scope: &str,
4866    ) {
4867        self.route_outages.record_accepted(module_id);
4868        self.counters.increment_route_open_accepted(principal);
4869        info!(
4870            target: "control",
4871            principal,
4872            module_id,
4873            connection_id = ctx.connection_id.get(),
4874            scope = %scope,
4875            "route.open accepted"
4876        );
4877    }
4878
4879    fn supervised_absent_route_open_refusal_frame(
4880        &self,
4881        ctx: &RouteCtx,
4882        caller: &RouteOpenCaller,
4883        frame: &Frame,
4884        module_id: &str,
4885        code: &'static str,
4886        status: &crate::supervise::ModuleStatus,
4887    ) -> Result<Frame, RouterError> {
4888        self.counters.increment_route_open_refused(code);
4889        info!(
4890            target: "control",
4891            code,
4892            reason = "supervised_not_registered",
4893            module_id = ?module_id,
4894            connection_id = ctx.connection_id.get(),
4895            principal = %caller.principal,
4896            scope = %caller.scope,
4897            harness = %caller.harness,
4898            state = %status.state,
4899            enabled = status.enabled,
4900            live = status.live,
4901            "route.open refused"
4902        );
4903        // A supervised module whose process has not registered is not
4904        // serving, whatever the reason; the supervisor knows this id, so it is
4905        // safe to track.
4906        self.route_outages
4907            .record_not_serving(module_id, "supervised_not_registered");
4908        let message = if let Some(pid) = status.waiting_for_previous_pid {
4909            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4910                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4911            })
4912        } else {
4913            format!(
4914                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4915                status.state, status.enabled, status.live
4916            )
4917        };
4918        control_error_frame(frame, code, message)
4919    }
4920
4921    async fn handle_route_open(
4922        &self,
4923        ctx: &RouteCtx,
4924        frame: Frame,
4925        request: RouteOpenRequest,
4926        timings: &mut RouteOpenTimings,
4927    ) -> Result<Vec<Frame>, RouterError> {
4928        let RouteOpenRequest {
4929            target,
4930            mut identity,
4931            consumer_identity,
4932            consumer_capabilities,
4933            role_versions,
4934            busy_refusals,
4935            admission_facts,
4936            scope,
4937        } = request;
4938        let target_module_id = target_module_id(&target).to_string();
4939        timings.module_id.clone_from(&target_module_id);
4940        if self
4941            .registry
4942            .get_module_by_connection(ctx.connection_id)
4943            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4944            .is_some()
4945        {
4946            return Ok(vec![control_error_frame(
4947                &frame,
4948                "invalid_request",
4949                "module connections cannot open client routes",
4950            )?]);
4951        }
4952        debug!(
4953            connection_id = ctx.connection_id.get(),
4954            corr = frame.header.corr,
4955            module_id = %target_module_id,
4956            "handling route.open"
4957        );
4958
4959        // The caller's attested principal, requested scope and harness, for
4960        // every `route.open refused` line below. The launch nonce is checked
4961        // here, once, and the answer is reused where the principal is
4962        // enforced further down, so the order of refusals is unchanged.
4963        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4964        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4965
4966        // A malformed declaration is refused first, before anything about the
4967        // target is looked up: the same body would be refused against any
4968        // module, so the caller learns nothing by retrying or waiting. An empty
4969        // map declares nothing and travels as no field at all, so a provider
4970        // only ever sees a missing field or a non-empty one.
4971        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4972        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4973            self.observe_route_open_refusal(
4974                ctx,
4975                &caller,
4976                &target_module_id,
4977                "invalid_role_versions",
4978                error_codes::INVALID_REQUEST,
4979            );
4980            return Ok(vec![control_error_body_frame(
4981                &frame,
4982                ErrorBody {
4983                    code: error_codes::INVALID_REQUEST.to_string(),
4984                    message: error.to_string(),
4985                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4986                },
4987            )?]);
4988        }
4989
4990        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4991            if status.waiting_for_previous_pid.is_some() {
4992                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4993                    ctx,
4994                    &caller,
4995                    &frame,
4996                    &target_module_id,
4997                    "module_warming",
4998                    &status,
4999                )?]);
5000            }
5001        }
5002
5003        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
5004        // opposite. Below, a caller learns whether a module is unregistered,
5005        // supervised-but-down (with state/enabled/live), or registered without the
5006        // requested role. Elsewhere that is an enumeration leak: a probe learning
5007        // the shape of a fleet it cannot otherwise see.
5008        //
5009        // It is not one here, and the reason is the ACCESS MODEL rather than
5010        // anything about these errors. Reaching route.open requires the
5011        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
5012        // connection file, so any caller who completes it already runs as this
5013        // user -- and can read subc.jsonc for the module list and `ck module
5014        // status` for live state. The reply discloses nothing the caller cannot
5015        // read more easily from disk, while the precision is load-bearing:
5016        // `unknown_module` is retryable and a missing role is not.
5017        //
5018        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
5019        // remote transport, a sandboxed caller, a shared-host mode -- THAT
5020        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
5021        let Some(registration) = self
5022            .registry
5023            .get_module(&target_module_id)
5024            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5025        else {
5026            if let Some((status, warming)) =
5027                self.supervisor_status(&target_module_id, frame.header.corr)?
5028            {
5029                // BEFORE the two availability codes below, because for a module
5030                // that speaks no subc wire both of them are false comfort: they
5031                // say "not right now" and are retried, and this module will
5032                // never register no matter how long the caller waits. The
5033                // absence here is the declaration being honoured, not a module
5034                // that is late.
5035                if status.protocol == ModuleProtocol::None {
5036                    return Ok(vec![self.route_open_refusal_frame(
5037                        ctx,
5038                        &caller,
5039                        &frame,
5040                        &target_module_id,
5041                        "protocol_none",
5042                        error_codes::MODULE_NO_PROTOCOL,
5043                        format!(
5044                            "module_id '{target_module_id}' is declared protocol: none; \
5045                             it speaks no subc wire and serves no routes"
5046                        ),
5047                    )?]);
5048                }
5049                let code = if warming {
5050                    "module_warming"
5051                } else {
5052                    "target_unavailable"
5053                };
5054                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5055                    ctx,
5056                    &caller,
5057                    &frame,
5058                    &target_module_id,
5059                    code,
5060                    &status,
5061                )?]);
5062            }
5063            if let Some(removed_ago_ms) =
5064                self.supervisor.removal_tombstone_age_ms(&target_module_id)
5065            {
5066                return Ok(vec![self.route_open_refusal_frame(
5067                    ctx,
5068                    &caller,
5069                    &frame,
5070                    &target_module_id,
5071                    "removed",
5072                    error_codes::MODULE_REMOVED,
5073                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5074                )?]);
5075            }
5076            return Ok(vec![self.route_open_refusal_frame(
5077                ctx,
5078                &caller,
5079                &frame,
5080                &target_module_id,
5081                "not_registered",
5082                error_codes::UNKNOWN_MODULE,
5083                format!("module_id '{target_module_id}' is not registered"),
5084            )?]);
5085        };
5086
5087        // Best-effort only: registry readiness and forwarding reservation use
5088        // different locks, so a module can flip readiness between this read and
5089        // the relay. Modules must still tolerate an `on_bind` while not ready.
5090        if !registration.ready {
5091            self.counters
5092                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5093            info!(
5094                target: "control",
5095                code = error_codes::MODULE_WARMING,
5096                module_id = ?target_module_id,
5097                connection_id = ctx.connection_id.get(),
5098                principal = %caller.principal,
5099                scope = %caller.scope,
5100                harness = %caller.harness,
5101                reason = "declared_not_ready",
5102                "route.open refused"
5103            );
5104            // The module is registered but says it cannot take work, which is
5105            // an outage from the caller's side even though its process is up.
5106            self.route_outages
5107                .record_not_serving(&target_module_id, "declared_not_ready");
5108            return Ok(vec![control_error_body_frame(
5109                &frame,
5110                ErrorBody {
5111                    code: error_codes::MODULE_WARMING.to_string(),
5112                    message: format!(
5113                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5114                    ),
5115                    detail: Some(serde_json::json!({
5116                        "reason": "declared_not_ready"
5117                    })),
5118                },
5119            )?]);
5120        }
5121
5122        // Effective readiness, second half: a module that declares a capability
5123        // `need: required` is not routable while that capability has no
5124        // registered provider. It is enforced HERE, as a retryable routing
5125        // refusal, and deliberately not as spawn ordering or a boot block. The
5126        // module is still started and registered and can make its own calls;
5127        // spawn ordering is a promise that cannot be kept once a provider
5128        // crashes at runtime, and refusing to boot would stop the whole
5129        // machine, including the tools needed to fix its configuration.
5130        //
5131        // "Provided" is the evaluator's verdict, which counts a provider as
5132        // soon as it has REGISTERED, not once it is ready. Two modules that
5133        // require each other's capabilities are therefore both routable once
5134        // both register; counting readiness instead would deadlock them.
5135        //
5136        // Only new opens are refused. Routes already bound when a provider
5137        // goes away stay bound: nothing here tears them down, and the module
5138        // answers them as it can. Like the readiness read above this is
5139        // best-effort against a provider registering or leaving concurrently.
5140        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
5141            self.counters
5142                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5143            info!(
5144                target: "control",
5145                code = error_codes::MODULE_WARMING,
5146                module_id = ?target_module_id,
5147                connection_id = ctx.connection_id.get(),
5148                principal = %caller.principal,
5149                scope = %caller.scope,
5150                harness = %caller.harness,
5151                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5152                capability = %capability,
5153                "route.open refused"
5154            );
5155            return Ok(vec![control_error_body_frame(
5156                &frame,
5157                ErrorBody {
5158                    code: error_codes::MODULE_WARMING.to_string(),
5159                    message: format!(
5160                        "module_id '{target_module_id}' requires capability '{capability}', \
5161                         which no registered module provides; retry"
5162                    ),
5163                    detail: Some(serde_json::json!({
5164                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5165                        "capability": capability,
5166                    })),
5167                },
5168            )?]);
5169        }
5170
5171        if !target_has_required_role(&target, &registration.manifest.provides) {
5172            return Ok(vec![self.route_open_refusal_frame(
5173                ctx,
5174                &caller,
5175                &frame,
5176                &target_module_id,
5177                "role_not_provided",
5178                "target_unavailable",
5179                format!("module_id '{target_module_id}' does not provide the requested target"),
5180            )?]);
5181        }
5182
5183        if registration.state != ChannelState::Active {
5184            return Ok(vec![self.route_open_refusal_frame(
5185                ctx,
5186                &caller,
5187                &frame,
5188                &target_module_id,
5189                "registration_not_active",
5190                "target_unavailable",
5191                format!("module_id '{target_module_id}' is not active"),
5192            )?]);
5193        }
5194
5195        if self
5196            .forwarding
5197            .module_is_draining(&target_module_id)
5198            .map_err(RouterError::Forwarding)?
5199        {
5200            return Ok(vec![self.route_open_refusal_frame(
5201                ctx,
5202                &caller,
5203                &frame,
5204                &target_module_id,
5205                "reloading",
5206                "module_reloading",
5207                format!("module_id '{target_module_id}' is reloading"),
5208            )?]);
5209        }
5210
5211        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5212            process_liveness.process_live(&target_module_id) == Some(false)
5213        }) {
5214            // A module the supervisor is restarting or reloading can still hold
5215            // a registration: the old process before its connection closes, or
5216            // a new one that registered while the supervisor was draining. The
5217            // forwarding table does not see that as draining, but the consumer
5218            // should still be told to retry soon, exactly as for the drain
5219            // above, rather than that the target is unavailable.
5220            if process_liveness.process_replacing(&target_module_id) {
5221                return Ok(vec![self.route_open_refusal_frame(
5222                    ctx,
5223                    &caller,
5224                    &frame,
5225                    &target_module_id,
5226                    "reloading",
5227                    "module_reloading",
5228                    format!("module_id '{target_module_id}' is reloading"),
5229                )?]);
5230            }
5231            return Ok(vec![self.route_open_refusal_frame(
5232                ctx,
5233                &caller,
5234                &frame,
5235                &target_module_id,
5236                "supervisor_not_live",
5237                "target_unavailable",
5238                format!("module_id '{target_module_id}' is not live"),
5239            )?]);
5240        }
5241
5242        if !self
5243            .forwarding
5244            .has_live_module_connection(&target_module_id)
5245            .map_err(RouterError::Forwarding)?
5246        {
5247            return Ok(vec![self.route_open_refusal_frame(
5248                ctx,
5249                &caller,
5250                &frame,
5251                &target_module_id,
5252                "no_forwarding_connection",
5253                "target_unavailable",
5254                format!("module_id '{target_module_id}' has no live forwarding connection"),
5255            )?]);
5256        }
5257
5258        if let Some(error) =
5259            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5260        {
5261            self.observe_route_open_refusal(
5262                ctx,
5263                &caller,
5264                &target_module_id,
5265                "op_not_allowed",
5266                "op_not_allowed",
5267            );
5268            return Ok(vec![error]);
5269        }
5270
5271        let principal =
5272            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5273                Ok(principal) => principal,
5274                Err(error) => {
5275                    self.observe_route_open_refusal(
5276                        ctx,
5277                        &caller,
5278                        &target_module_id,
5279                        "bad_consumer_identity",
5280                        "bad_consumer_identity",
5281                    );
5282                    return Ok(vec![error]);
5283                }
5284            };
5285        timings.principal = principal_label(&principal);
5286
5287        // Only Callosum can attest a production phone build. Consume the claim
5288        // here so it can be kept on the route without reaching any module.
5289        let build_class = identity.build_class.take().filter(|build_class| {
5290            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5291                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5292            } else {
5293                if self
5294                    .build_class_warned
5295                    .lock()
5296                    .unwrap_or_else(|p| p.into_inner())
5297                    .insert(ctx.connection_id)
5298                {
5299                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5300                        "ignoring build_class from a non-Callosum opener");
5301                }
5302                false
5303            }
5304        });
5305
5306        // This is attested, control-plane policy for supervised module origins.
5307        // Keep it before route reservation and out of the opaque forwarding hot
5308        // path: data frames must never acquire a per-frame capability check.
5309        if let Principal::Reserved {
5310            module_id: opening_module_id,
5311        } = &principal
5312        {
5313            if let Some(opening_registration) = self
5314                .registry
5315                .get_module(opening_module_id)
5316                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5317            {
5318                if let Some(capability) =
5319                    denied_capability(&opening_registration.manifest, &registration.manifest)
5320                {
5321                    warn!(
5322                        opening_module_id,
5323                        target_module_id,
5324                        capability,
5325                        "refusing route.open because an attested capability deny edge matches"
5326                    );
5327                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5328                        &frame,
5329                        &target_module_id,
5330                        "capability_deny_edge",
5331                        "capability_forbidden",
5332                        format!(
5333                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5334                        ),
5335                    )?]);
5336                }
5337            }
5338        }
5339
5340        if admission_facts.is_some() {
5341            let carrier_matches = matches!(
5342                &principal,
5343                Principal::Reserved { module_id }
5344                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5345            );
5346            if !carrier_matches {
5347                return Ok(vec![self.route_open_refusal_frame(
5348                    ctx,
5349                    &caller,
5350                    &frame,
5351                    &target_module_id,
5352                    "admission_facts_carrier_not_permitted",
5353                    "admission_facts_not_permitted",
5354                    "admission facts may only be carried by the configured reserved module",
5355                )?]);
5356            }
5357
5358            let target_allowed = self
5359                .admission_facts_targets
5360                .as_ref()
5361                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5362            if !target_allowed {
5363                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5364                    &frame,
5365                    &target_module_id,
5366                    "admission_facts_target_not_listed",
5367                    "admission_facts_target_not_allowed",
5368                    format!(
5369                        "admission facts are not permitted for target module_id '{target_module_id}'"
5370                    ),
5371                )?]);
5372            }
5373
5374            // Keep the value opaque to subc. The downstream admission validator owns
5375            // schema and semantic checks; this daemon only enforces carrier authority
5376            // and the configured destination allowlist.
5377        }
5378
5379        // Scope admission, on the attested principal above and never on the
5380        // request body. The tag read here travels with the pending bind and is
5381        // compared with the published one at commit, so a sync between here
5382        // and the module's ack refuses the open instead of binding a stamp
5383        // that is no longer true.
5384        let (bound_scope, mut scope_stamp) = match &scope {
5385            None => (None, None),
5386            Some(selector) => {
5387                let owner_configured = match &selector.owner {
5388                    // A reserved owner counts as configured from before its
5389                    // process is spawned (see `SupervisorHandle::is_configured`).
5390                    // So an owner that has not synced its scopes yet is refused
5391                    // as retryable (`scope_not_synced`), not as one that will
5392                    // never sync.
5393                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5394                    _ => false,
5395                };
5396                let mut scopes = self
5397                    .scopes
5398                    .write()
5399                    .unwrap_or_else(|poisoned| poisoned.into_inner());
5400                let admitted = scopes
5401                    .admit(&principal, &target_module_id, selector, owner_configured)
5402                    .and_then(|admission| {
5403                        crate::scopes::check_target_flow_support(
5404                            &admission.stamp,
5405                            &target_module_id,
5406                            registration.manifest.capabilities.as_ref(),
5407                        )?;
5408                        crate::scopes::check_target_agent_run_support(
5409                            &admission.stamp,
5410                            &target_module_id,
5411                            registration.manifest.capabilities.as_ref(),
5412                        )?;
5413                        let bound = scopes.reserve_bind(
5414                            &admission,
5415                            &principal,
5416                            ctx.connection_id,
5417                            selector.hold_presence,
5418                        );
5419                        Ok((admission, bound))
5420                    });
5421                match admitted {
5422                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5423                    Err(refusal) => {
5424                        return Ok(vec![self.route_open_refusal_frame(
5425                            ctx,
5426                            &caller,
5427                            &frame,
5428                            &target_module_id,
5429                            refusal.code,
5430                            refusal.code,
5431                            refusal.message,
5432                        )?]);
5433                    }
5434                }
5435            }
5436        };
5437        // From here on a refusal names the scope by the stamp admission
5438        // issued, exactly as the `route.open accepted` line does.
5439        if let Some(stamp) = &scope_stamp {
5440            caller.scope = route_open_scope_label(stamp);
5441        }
5442
5443        // Allow resolvable missing-tail paths so a vanished directory does not
5444        // prevent reopening an existing session for cleanup. Binding does not
5445        // authorize creation of durable provider state.
5446        //
5447        // Resolve real paths rather than lexical cleanup: canonicalize the
5448        // longest existing ancestor and append the missing tail, retaining the
5449        // path's real existing ancestry even when a leaf is missing.
5450        timings.enter(RouteOpenPhase::ProjectRoot);
5451        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5452            Ok(project) => project,
5453            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5454        };
5455        timings.enter(RouteOpenPhase::Admission);
5456        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5457            Ok(root) => root,
5458            Err(err) => {
5459                return Ok(vec![control_error_frame(
5460                    &frame,
5461                    "invalid_project_root",
5462                    err.to_string(),
5463                )?])
5464            }
5465        };
5466        // Compute compatibility project fields before stripping optional stamp
5467        // members. Preserve the canonical opener root and its project ID,
5468        // including absence. For rootless openers, use stamp workdir or `/`, and
5469        // opener ID, else stamp ID, else no ID. Emit these identity fields only
5470        // to targets without bind-no-project/v1.
5471        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5472
5473        // Last gate before any relay work, and deliberately after the cheap
5474        // registry and availability checks above: those name a more precise
5475        // condition (unknown, removed, reloading) and a caller is better served
5476        // by the precise code than by this one.
5477        //
5478        // Everything below this point costs an egress permit, a reserved handle
5479        // pair and, if the module does not answer, the whole relay budget. The
5480        // reader no longer waits for that budget, so cap each target explicitly;
5481        // serial dispatch used to provide the accidental cap of one relay per
5482        // connection. Admission is a mutex-protected count and never waits.
5483        let _concurrency_guard = match self
5484            .route_bind_concurrency
5485            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5486        {
5487            Ok(guard) => guard,
5488            Err(in_flight) => {
5489                return Ok(vec![self.route_open_target_capacity_refusal(
5490                    ctx,
5491                    &caller,
5492                    &frame,
5493                    &target_module_id,
5494                    in_flight,
5495                )?]);
5496            }
5497        };
5498
5499        // A module that has already burned the whole budget `threshold` times
5500        // in a row does not get to charge it again until a probe says it recovered.
5501        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5502            RouteBindAdmission::Admitted { guard, probe } => {
5503                if probe {
5504                    info!(
5505                        module_id = %target_module_id,
5506                        connection_id = ctx.connection_id.get(),
5507                        "route.bind breaker half-open: admitting one probe"
5508                    );
5509                }
5510                guard
5511            }
5512            RouteBindAdmission::Refused {
5513                consecutive_timeouts,
5514                retry_in,
5515                probe_in_flight,
5516            } => {
5517                return Ok(vec![self.route_open_breaker_refusal_frame(
5518                    ctx,
5519                    &caller,
5520                    &frame,
5521                    &target_module_id,
5522                    consecutive_timeouts,
5523                    retry_in,
5524                    probe_in_flight,
5525                )?]);
5526            }
5527        };
5528
5529        // Resolve the per-module budget here so the wait matches the operator's
5530        // intent for this specific target. A per-module override in
5531        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5532        // daemons) wins over the daemon-wide default.
5533        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5534        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5535        let operator_app = self
5536            .operator_apps
5537            .lock()
5538            .unwrap_or_else(|p| p.into_inner())
5539            .get(&ctx.connection_id)
5540            .cloned();
5541        timings.enter(RouteOpenPhase::EgressReservation);
5542        let pending = match self
5543            .forwarding
5544            .begin_route_bind_relay_for(
5545                ctx.connection_id,
5546                ctx.egress.clone(),
5547                response_version(&frame),
5548                frame.header.corr,
5549                &target_module_id,
5550                principal.clone(),
5551                bound_scope,
5552                project_root,
5553                identity.harness.clone(),
5554                build_class,
5555                operator_app.clone(),
5556                relay_deadline,
5557                busy_refusals == Some(true),
5558            )
5559            .await
5560        {
5561            Ok(pending) => pending,
5562            Err(err) => {
5563                return Ok(vec![self.route_open_refusal_frame(
5564                    ctx,
5565                    &caller,
5566                    &frame,
5567                    &target_module_id,
5568                    "relay_reservation_failed",
5569                    forwarding_error_code(&err),
5570                    err.to_string(),
5571                )?])
5572            }
5573        };
5574        timings.enter(RouteOpenPhase::Admission);
5575        let crate::forwarding::PendingRouteBindRelay {
5576            endpoint,
5577            module_sink,
5578            negotiated_ver,
5579            client_channel,
5580            client_epoch,
5581            module_channel,
5582            module_epoch,
5583            corr: relay_corr,
5584            receiver,
5585        } = pending;
5586        let mut reservation =
5587            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5588
5589        // Reserving egress can wait while a module reconnects or a swap cuts
5590        // over. Check the relay's captured connection, not the earlier by-id
5591        // lookup: the original module's flow/run capabilities cannot authorize
5592        // its replacement. The captured sink stays bound to that connection.
5593        let relay_registration = self
5594            .registry
5595            .get_module_by_connection(endpoint.connection_id)
5596            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5597        let relay_capabilities = relay_registration
5598            .as_ref()
5599            .and_then(|registration| registration.manifest.capabilities.as_ref());
5600        if let Some(selector) = &scope {
5601            // Egress reservation may wait across a scope update. Recheck the
5602            // proof against the current record before sending the captured bind.
5603            if let Err(refusal) = self
5604                .scopes
5605                .read()
5606                .unwrap_or_else(|poisoned| poisoned.into_inner())
5607                .check_bind_proof(&principal, selector)
5608            {
5609                reservation.release_and_disarm();
5610                return Ok(vec![self.route_open_refusal_frame(
5611                    ctx,
5612                    &caller,
5613                    &frame,
5614                    &target_module_id,
5615                    refusal.code,
5616                    refusal.code,
5617                    refusal.message,
5618                )?]);
5619            }
5620        }
5621        if let Some(stamp) = scope_stamp.as_ref() {
5622            if let Err(refusal) = crate::scopes::check_target_flow_support(
5623                stamp,
5624                &target_module_id,
5625                relay_capabilities,
5626            )
5627            .and_then(|()| {
5628                crate::scopes::check_target_agent_run_support(
5629                    stamp,
5630                    &target_module_id,
5631                    relay_capabilities,
5632                )
5633            }) {
5634                reservation.release_and_disarm();
5635                return Ok(vec![self.route_open_refusal_frame(
5636                    ctx,
5637                    &caller,
5638                    &frame,
5639                    &target_module_id,
5640                    refusal.code,
5641                    refusal.code,
5642                    refusal.message,
5643                )?]);
5644            }
5645        }
5646
5647        // Older readers may reject unknown fields. Select optional fields from
5648        // the captured endpoint's declaration, including on unscoped binds.
5649        if scope_stamp
5650            .as_ref()
5651            .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5652            && !crate::scopes::target_supports_federated(relay_capabilities)
5653        {
5654            // Keep bound_scope: even an unstamped legacy route belongs to this epoch.
5655            scope_stamp = None;
5656        }
5657        if !relay_capabilities.is_some_and(|caps| {
5658            caps.provides
5659                .iter()
5660                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5661        }) {
5662            if let Some(stamp) = scope_stamp.as_mut() {
5663                stamp.workdir = None;
5664                stamp.project_id = None;
5665            }
5666        }
5667        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5668        // attribution. The run gate above has already required the target to
5669        // declare `worker-runs/v1`, which promises to read both
5670        // `on_behalf_of_agent` and `call_mode`.
5671        let worker_run = scope_stamp
5672            .as_ref()
5673            .is_some_and(crate::scopes::is_worker_run)
5674            && relay_capabilities.is_some_and(|caps| {
5675                caps.provides
5676                    .iter()
5677                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5678            });
5679        if !worker_run
5680            && !relay_capabilities.is_some_and(|caps| {
5681                caps.provides
5682                    .iter()
5683                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5684            })
5685        {
5686            if let Some(stamp) = scope_stamp.as_mut() {
5687                stamp.attributes.on_behalf_of_agent = None;
5688            }
5689        }
5690        if !relay_capabilities.is_some_and(|caps| {
5691            caps.provides
5692                .iter()
5693                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5694        }) {
5695            if let Some(stamp) = scope_stamp.as_mut() {
5696                stamp.attributes.flow_version = None;
5697                stamp.attributes.flow_code_hash = None;
5698                stamp.attributes.flow_install_id = None;
5699            }
5700        }
5701        let call_mode = if worker_run
5702            || relay_capabilities
5703                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5704        {
5705            Some(derive_call_mode(scope_stamp.as_ref()))
5706        } else {
5707            if let Some(stamp) = scope_stamp.as_mut() {
5708                stamp.expires_at_ms = None;
5709            }
5710            None
5711        };
5712
5713        debug!(
5714            connection_id = ctx.connection_id.get(),
5715            client_channel,
5716            client_epoch,
5717            module_channel,
5718            module_epoch,
5719            "reserved route handle pair"
5720        );
5721        // Rendered BEFORE the move into the relay, because the accept arm below
5722        // is where it is logged and the principal is gone by then.
5723        let route_principal_label = principal_label(&principal);
5724        let relay = ModuleControlRequest::RouteBind {
5725            route_channel: module_channel,
5726            epoch: module_epoch,
5727            target,
5728            identity,
5729            principal: Some(principal),
5730            consumer_capabilities,
5731            role_versions,
5732            admission_facts,
5733            scope: scope_stamp,
5734            call_mode,
5735        };
5736        let peer = RouteBindPeer::new()
5737            .with_observed_peer(
5738                relay_capabilities
5739                    .is_some_and(|caps| {
5740                        caps.provides
5741                            .iter()
5742                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5743                    })
5744                    .then(|| {
5745                        self.observed_peers
5746                            .lock()
5747                            .unwrap_or_else(|p| p.into_inner())
5748                            .get(&ctx.connection_id)
5749                            .cloned()
5750                            .unwrap_or_else(ObservedPeer::unobserved)
5751                    }),
5752            )
5753            .with_operator_app(
5754                self.connection_operator_app(operator_app.as_ref())
5755                    .filter(|_| {
5756                        relay_capabilities.is_some_and(|caps| {
5757                            caps.provides
5758                                .iter()
5759                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5760                        })
5761                    }),
5762            );
5763        let relay_body = crate::bind_compat::encode_bind(
5764            &relay,
5765            &peer,
5766            &legacy_bind_project,
5767            relay_capabilities,
5768        )
5769        .map_err(|err| {
5770            RouterError::backend(
5771                0,
5772                frame.header.corr,
5773                format!("failed to encode route.bind request: {err}"),
5774            )
5775        })?;
5776        let relay_frame = Frame::build_with_version(
5777            negotiated_ver,
5778            FrameType::Request,
5779            control_flags(),
5780            0,
5781            0,
5782            relay_corr,
5783            relay_body,
5784        )
5785        .map_err(RouterError::FrameBuild)?;
5786
5787        timings.enter(RouteOpenPhase::BindSend);
5788        let sent = module_sink.send(relay_frame).await;
5789        timings.enter(RouteOpenPhase::Admission);
5790        if let Err(err) = sent {
5791            reservation.release_and_disarm();
5792            return Ok(vec![self.route_open_refusal_frame(
5793                ctx,
5794                &caller,
5795                &frame,
5796                &target_module_id,
5797                "relay_send_failed",
5798                "target_unavailable",
5799                err.to_string(),
5800            )?]);
5801        }
5802
5803        #[cfg(test)]
5804        if let Some(hold) = &self.route_bind_relay_sent_hold {
5805            // First wait: tell the test the bind is queued. Second wait: stay
5806            // here until the test has driven whatever it wants to happen
5807            // between queueing the bind and marking it enqueued.
5808            hold.wait().await;
5809            hold.wait().await;
5810        }
5811
5812        // A missing relay is not by itself an abandoned bind: the module may
5813        // already have acked it and the route may be committed. Only a relay
5814        // dropped without any module reply needs this GOODBYE; one a reply
5815        // settled has already had its GOODBYE decided by the reply path.
5816        if self
5817            .forwarding
5818            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5819            .map_err(RouterError::Forwarding)?
5820            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5821        {
5822            self.send_abandoned_route_bind_goodbye(
5823                &module_sink,
5824                negotiated_ver,
5825                module_channel,
5826                module_epoch,
5827            );
5828        }
5829
5830        timings.enter(RouteOpenPhase::BindAckWait);
5831        let outcome = timeout_at(relay_deadline, receiver).await;
5832        timings.enter(RouteOpenPhase::Admission);
5833        match outcome {
5834            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5835                reservation.disarm();
5836                if breaker.record_accepted() {
5837                    info!(
5838                        module_id = %target_module_id,
5839                        "route.bind breaker closed: the probe was accepted"
5840                    );
5841                }
5842                self.observe_route_open_accept(
5843                    ctx,
5844                    &target_module_id,
5845                    &route_principal_label,
5846                    &caller.scope,
5847                );
5848                Ok(Vec::new())
5849            }
5850            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5851                reservation.release_and_disarm();
5852                // A module that says no in microseconds is healthy. Rejection
5853                // is a different condition with its own refusal and must not
5854                // move the breaker.
5855                breaker.record_inconclusive();
5856                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5857            }
5858            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5859                reservation.release_and_disarm();
5860                breaker.record_inconclusive();
5861                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5862            }
5863            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5864                reservation.release_and_disarm();
5865                breaker.record_inconclusive();
5866                // Fires when the module's connection closes while a relayed
5867                // bind is pending -- typically a caller racing a module restart
5868                // whose bind was relayed BEFORE the drain mark went up. Logged
5869                // because the caller sees only its own error and the fleet has
5870                // already spent one diagnosis round unable to tell this arm
5871                // from a relay timeout without daemon-side evidence.
5872                tracing::warn!(
5873                    module_id = %target_module_id,
5874                    "route.bind relay abandoned: {message}"
5875                );
5876                Ok(vec![self.route_open_refusal_frame(
5877                    ctx,
5878                    &caller,
5879                    &frame,
5880                    &target_module_id,
5881                    "relay_abandoned",
5882                    "target_unavailable",
5883                    message,
5884                )?])
5885            }
5886            Ok(Err(_)) => {
5887                reservation.release_and_disarm();
5888                breaker.record_inconclusive();
5889                Ok(vec![self.route_open_refusal_frame(
5890                    ctx,
5891                    &caller,
5892                    &frame,
5893                    &target_module_id,
5894                    "relay_waiter_canceled",
5895                    "target_unavailable",
5896                    "route.bind relay waiter was canceled before the module responded",
5897                )?])
5898            }
5899            Err(_) => {
5900                reservation.release_and_disarm();
5901                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5902                // answer at all is the one condition a fast refusal can
5903                // usefully stand in for; every other arm already answered.
5904                if let Some(opened) = breaker.record_timeout(
5905                    self.route_bind_breaker_threshold,
5906                    self.route_bind_breaker_cooldown,
5907                ) {
5908                    warn!(
5909                        module_id = %target_module_id,
5910                        consecutive_timeouts = opened.consecutive_timeouts,
5911                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5912                        reopened_after_probe = opened.reopened_after_probe,
5913                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5914                    );
5915                }
5916                // The generous budget just burned to no answer: the module is
5917                // registered and its connection is up, but its bind handler sat
5918                // on the ack for the full budget (warm-on-bind, cold configure,
5919                // or a wedged handler). Every earlier unavailability shape
5920                // fast-refuses BEFORE the relay, so this arm firing means the
5921                // slowness is module-side -- log it so the per-module timeline
5922                // is reconstructable without client audit rows.
5923                tracing::warn!(
5924                    module_id = %target_module_id,
5925                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5926                    "route.bind relay timed out: module did not ack within budget"
5927                );
5928                Ok(vec![self.route_open_refusal_frame(
5929                    ctx,
5930                    &caller,
5931                    &frame,
5932                    &target_module_id,
5933                    "relay_timed_out",
5934                    "module_timeout",
5935                    format!(
5936                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5937                        route_bind_relay_timeout
5938                    ),
5939                )?])
5940            }
5941        }
5942    }
5943
5944    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5945        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5946            snapshot: self.supervisor.spawn_snapshot(),
5947        };
5948        Ok(vec![control_response_body_frame(
5949            &frame,
5950            &response,
5951            "ClientControlResponse::SupervisorSpawnSnapshot",
5952        )?])
5953    }
5954
5955    fn handle_supervisor_spawn_subscribe(
5956        &self,
5957        ctx: &RouteCtx,
5958        frame: Frame,
5959        since: Option<SpawnCursor>,
5960    ) -> Result<Vec<Frame>, RouterError> {
5961        match self.supervisor.subscribe_spawns(
5962            ctx.connection_id,
5963            frame.header.corr,
5964            response_version(&frame),
5965            since,
5966            ctx.egress.clone(),
5967        ) {
5968            Ok(()) => Ok(Vec::new()),
5969            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5970                Ok(vec![control_error_body_frame(
5971                    &frame,
5972                    ErrorBody {
5973                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5974                        message: "spawn cursor belongs to a different daemon incarnation"
5975                            .to_string(),
5976                        detail: Some(serde_json::json!({
5977                            "current_daemon_incarnation": current
5978                        })),
5979                    },
5980                )?])
5981            }
5982            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5983                &frame,
5984                ErrorBody {
5985                    code: "spawn_cursor_too_old".to_string(),
5986                    message: "spawn cursor predates the retained event ring".to_string(),
5987                    detail: Some(serde_json::json!({
5988                        "oldest_retained_cursor": oldest
5989                    })),
5990                },
5991            )?]),
5992            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5993                0,
5994                frame.header.corr,
5995                format!("failed to open supervisor spawn subscription: {error}"),
5996            )),
5997        }
5998    }
5999
6000    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6001        // Status/configuration snapshots may wait on synchronous supervisor
6002        // locks. Keep that work off the connection's async runtime thread.
6003        let handler = self.clone();
6004        let runtime = tokio::runtime::Handle::current();
6005        // The blocking pool does not inherit the control request's tracing
6006        // subscriber. Preserve it so slow snapshot-lock warnings stay visible.
6007        let subscriber = tracing::dispatcher::get_default(Clone::clone);
6008        let corr = frame.header.corr;
6009        tokio::task::spawn_blocking(move || {
6010            runtime.block_on(
6011                list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6012                    .with_subscriber(subscriber),
6013            )
6014        })
6015        .await
6016        .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6017    }
6018
6019    async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6020        let generation = self
6021            .registry
6022            .generation()
6023            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6024        let supervised = self.supervisor.list();
6025        let mut statuses = BTreeMap::new();
6026        let mut not_running = BTreeSet::new();
6027        for module in &supervised {
6028            let status = module.status_for_control("list").map_err(|err| {
6029                RouterError::backend(
6030                    0,
6031                    frame.header.corr,
6032                    format!("failed to read supervisor status: {err}"),
6033                )
6034            })?;
6035            if !status.enabled || !status.process_alive {
6036                not_running.insert(module.module_id().to_owned());
6037            }
6038            statuses.insert(module.module_id().to_owned(), status);
6039        }
6040        // Legacy per-process diagnostics keep their shared optional budget.
6041        // The cached tree and accounting export below does not spend it.
6042        let mut process_resources = BTreeMap::new();
6043        for module in &supervised {
6044            process_resources.insert(
6045                module.module_id().to_owned(),
6046                crate::subtree_resources::read_process(module.clone()).await,
6047            );
6048        }
6049        // Image evidence uses the same optional deadline, but an exhausted
6050        // budget must not suppress the committed resource view.
6051        let mut image_readings = BTreeMap::new();
6052        for module in &supervised {
6053            image_readings.insert(
6054                module.module_id().to_owned(),
6055                module.running_image_agreement().await,
6056            );
6057        }
6058        let readings = self.resource_tracker.list(supervised.clone());
6059        let mut modules = Vec::new();
6060        for module in supervised {
6061            let status = statuses
6062                .remove(module.module_id())
6063                .expect("each supervised module has a status snapshot");
6064            let (configured, _) = module.configuration().map_err(|err| {
6065                RouterError::backend(
6066                    0,
6067                    frame.header.corr,
6068                    format!("failed to read module configuration: {err}"),
6069                )
6070            })?;
6071            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6072                RouterError::backend(
6073                    0,
6074                    frame.header.corr,
6075                    format!("failed to read retry time: {err}"),
6076                )
6077            })?;
6078            let image = image_readings
6079                .remove(module.module_id())
6080                .expect("each supervised module has an image snapshot");
6081            let resources = process_resources.remove(module.module_id());
6082            let mut verdict =
6083                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6084            verdict.spec = Some(launch_spec_agreement(
6085                &configured,
6086                status.process_alive,
6087                status.launch_spec_digest.as_ref(),
6088            ));
6089            let pending_reload = Some(verdict);
6090            let configured_enabled = self
6091                .capability_evaluator
6092                .configured_enabled(&status.module_id);
6093            // Keep the retired policy field on the wire for one release so
6094            // existing status consumers still receive the platform policy.
6095            modules.push(
6096                SupervisorEntry::new(
6097                    status.module_id,
6098                    status.state.to_string(),
6099                    status.enabled,
6100                    status.live,
6101                    status.health.status,
6102                )
6103                .with_configured_enabled(configured_enabled)
6104                .with_next_retry_at_ms(next_retry_at_ms)
6105                .with_launch_nonce_env(Some(!cfg!(unix)))
6106                .with_launch_nonce_source(status.launch_nonce_source)
6107                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6108                .with_protocol(status.protocol)
6109                .with_pending_reload(pending_reload)
6110                .with_last_probe_ms(status.health.last_probe_ms)
6111                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6112                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6113                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6114                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6115                .with_restart_count(Some(status.restart_count))
6116                .with_max_restarts(Some(status.max_restarts))
6117                .with_lifetime_restarts(Some(status.lifetime_restarts))
6118                .with_spawn_generation(Some(status.spawn_generation))
6119                .with_restart_window_secs(Some(status.restart_window.as_secs()))
6120                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6121                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6122                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6123                .with_resources(resources),
6124            );
6125        }
6126        for module in &mut modules {
6127            if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6128                module.tree = tree.clone();
6129                module.cpu_last_hour = hour.clone();
6130            }
6131            module.tree_unavailable = readings
6132                .reasons
6133                .get(&module.module_id)
6134                .map(|reason| (*reason).to_owned());
6135            if module.tree.is_none() {
6136                module.tree_unavailable = Some(
6137                    if not_running.contains(&module.module_id) {
6138                        "not_running"
6139                    } else if let Some(reason) = readings.reasons.get(&module.module_id) {
6140                        reason
6141                    } else {
6142                        "not_yet_measured"
6143                    }
6144                    .to_owned(),
6145                );
6146            }
6147        }
6148        let response = ClientControlResponse::SupervisorList {
6149            generation,
6150            modules,
6151            daemon: readings.daemon,
6152            agent_commands: readings.agent_commands,
6153        };
6154        Ok(vec![control_response_body_frame(
6155            &frame,
6156            &response,
6157            "ClientControlResponse::SupervisorList",
6158        )?])
6159    }
6160
6161    fn handle_supervisor_stderr_tail(
6162        &self,
6163        frame: Frame,
6164        module_id: String,
6165        max_lines: Option<u32>,
6166        max_bytes: Option<u32>,
6167    ) -> Result<Vec<Frame>, RouterError> {
6168        let Some(module) = self.supervisor.get(&module_id) else {
6169            return Ok(vec![control_error_frame(
6170                &frame,
6171                "unknown_module",
6172                format!("module_id '{module_id}' is not supervised"),
6173            )?]);
6174        };
6175
6176        let snapshot = module.stderr_tail(
6177            max_lines.map(|value| value as usize),
6178            max_bytes.map(|value| value as usize),
6179        );
6180
6181        let response = ClientControlResponse::SupervisorStderrTail {
6182            module_id,
6183            tail: StderrTail {
6184                capture: match snapshot.capture {
6185                    CaptureState::Captured => StderrCaptureState::Captured,
6186                    CaptureState::Incomplete { reason } => {
6187                        StderrCaptureState::Incomplete { reason }
6188                    }
6189                    CaptureState::NotCaptured { reason } => {
6190                        StderrCaptureState::NotCaptured { reason }
6191                    }
6192                },
6193                entries: snapshot
6194                    .entries
6195                    .into_iter()
6196                    .map(|entry| match entry {
6197                        TailEntry::Line {
6198                            text,
6199                            truncated,
6200                            at_ms,
6201                        } => StderrTailEntry::Line {
6202                            text,
6203                            truncated,
6204                            at_ms,
6205                        },
6206                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6207                    })
6208                    .collect(),
6209                dropped_lines: snapshot.dropped_lines,
6210            },
6211        };
6212        Ok(vec![control_response_body_frame(
6213            &frame,
6214            &response,
6215            "ClientControlResponse::SupervisorStderrTail",
6216        )?])
6217    }
6218
6219    async fn observe_control_wait<F: std::future::Future>(
6220        &self,
6221        ctx: &RouteCtx,
6222        target: Option<&str>,
6223        kind: &'static str,
6224        future: F,
6225    ) -> F::Output {
6226        self.counters
6227            .observe_reader_wait(
6228                || {
6229                    let module = self
6230                        .forwarding
6231                        .module_id_for_connection(ctx.connection_id)
6232                        .ok()
6233                        .flatten();
6234                    let principal = if module.is_some() { "module" } else { "direct" };
6235                    (
6236                        self.resolve_reader_wait_target(kind, target)
6237                            .or(module)
6238                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6239                        principal.into(),
6240                    )
6241                },
6242                kind,
6243                future,
6244            )
6245            .await
6246    }
6247
6248    /// Resolve the reader-wait module ID only from supervisor or registry
6249    /// records; never attribute a wait from the request selector alone.
6250    pub(crate) fn resolve_reader_wait_target(
6251        &self,
6252        op: &str,
6253        selector: Option<&str>,
6254    ) -> Option<String> {
6255        let selector = selector?;
6256        match op {
6257            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6258                .supervisor
6259                .get(selector)
6260                .map(|module| module.module_id().to_owned()),
6261            ops::SUPERVISOR_HEALTH_PROBE => self
6262                .registry
6263                .get_module(selector)
6264                .ok()
6265                .flatten()
6266                .map(|registration| registration.manifest.module_id),
6267            _ => None,
6268        }
6269    }
6270
6271    async fn handle_supervisor_terminals(
6272        &self,
6273        ctx: &RouteCtx,
6274        frame: Frame,
6275        module_id: String,
6276    ) -> Result<Vec<Frame>, RouterError> {
6277        let Some(module) = self.supervisor.get(&module_id) else {
6278            return Ok(vec![control_error_frame(
6279                &frame,
6280                "unknown_module",
6281                format!("module_id '{module_id}' is not supervised"),
6282            )?]);
6283        };
6284
6285        // The journal read runs on a blocking thread: it can be megabytes of
6286        // file I/O and must not occupy a runtime worker.
6287        let terminals = self
6288            .observe_control_wait(
6289                ctx,
6290                Some(module.module_id()),
6291                ops::SUPERVISOR_TERMINALS,
6292                module.read_durable_terminal_history(),
6293            )
6294            .await
6295            .map_err(|error| {
6296                RouterError::backend(
6297                    0,
6298                    frame.header.corr,
6299                    format!("failed to read terminal history: {error}"),
6300                )
6301            })?;
6302        let response = ClientControlResponse::SupervisorTerminals {
6303            module_id,
6304            terminals,
6305        };
6306        Ok(vec![control_response_body_frame(
6307            &frame,
6308            &response,
6309            "ClientControlResponse::SupervisorTerminals",
6310        )?])
6311    }
6312
6313    fn handle_supervisor_routes(
6314        &self,
6315        frame: Frame,
6316        module_id: Option<String>,
6317    ) -> Result<Vec<Frame>, RouterError> {
6318        let modules = self
6319            .forwarding
6320            .route_census(module_id.as_deref())
6321            .map_err(RouterError::Forwarding)?
6322            .into_iter()
6323            .map(|(module_id, routes)| SupervisorRouteModule {
6324                module_id,
6325                routes: routes
6326                    .into_iter()
6327                    .map(|route| SupervisorRoute {
6328                        consumer: match route.principal {
6329                            Principal::Reserved { module_id } => {
6330                                SupervisorRouteConsumer::Reserved { module_id }
6331                            }
6332                            Principal::Direct | Principal::Unverified => {
6333                                SupervisorRouteConsumer::Direct {
6334                                    connection_id: route.goodbye_target.connection_id.get(),
6335                                }
6336                            }
6337                        },
6338                        age_ms: Instant::now()
6339                            .saturating_duration_since(route.bound_at)
6340                            .as_millis()
6341                            .try_into()
6342                            .unwrap_or(u64::MAX),
6343                        draining: route.draining,
6344                        drain_reason: route.drain_reason,
6345                    })
6346                    .collect(),
6347            })
6348            .collect();
6349        let response = ClientControlResponse::SupervisorRoutes { modules };
6350        Ok(vec![control_response_body_frame(
6351            &frame,
6352            &response,
6353            "ClientControlResponse::SupervisorRoutes",
6354        )?])
6355    }
6356
6357    async fn handle_supervisor_provenance(
6358        &self,
6359        ctx: &RouteCtx,
6360        frame: Frame,
6361        module_id: Option<String>,
6362    ) -> Result<Vec<Frame>, RouterError> {
6363        let targeted = module_id.is_some();
6364        let selected = if let Some(module_id) = module_id {
6365            let Some(module) = self.supervisor.get(&module_id) else {
6366                return Ok(vec![control_error_frame(
6367                    &frame,
6368                    "unknown_module",
6369                    format!("module_id '{module_id}' is not supervised"),
6370                )?]);
6371            };
6372            vec![module]
6373        } else {
6374            self.supervisor.list()
6375        };
6376
6377        let target = targeted.then(|| selected[0].module_id());
6378        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6379            let mut modules = Vec::with_capacity(selected.len());
6380            for module in &selected {
6381                let (status, observed_image) = module
6382                    .status_and_running_image_agreement()
6383                    .await
6384                    .map_err(|err| {
6385                        RouterError::backend(
6386                            0,
6387                            frame.header.corr,
6388                            format!("failed to read supervisor status: {err}"),
6389                        )
6390                    })?;
6391                let module_declared = self
6392                    .registry
6393                    .get_module(&status.module_id)
6394                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6395                    .and_then(|registration| registration.manifest.provenance)
6396                    .map(|build| ModuleDeclaredProvenance::Reported { build })
6397                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6398                #[cfg(test)]
6399                let running_image = match &self.provenance_probe_override {
6400                    Some(result) => result.clone(),
6401                    None => observed_image,
6402                };
6403                #[cfg(not(test))]
6404                let running_image = observed_image;
6405                modules.push(SupervisorModuleProvenance {
6406                    module_id: status.module_id,
6407                    module_declared,
6408                    daemon_observed: SupervisorObservedProcess {
6409                        launch_nonce_source: status.launch_nonce_source,
6410                        pid: status.pid,
6411                        spawned_at_ms: status.spawned_at_ms,
6412                        spawned_from: status.spawned_from,
6413                        running_image,
6414                    },
6415                });
6416            }
6417            let daemon = SupervisorDaemonProvenance {
6418                daemon_build: self.daemon_provenance.build.clone(),
6419                daemon_observed: DaemonObservedProcess {
6420                    pid: self.daemon_provenance.pid,
6421                    started_at_ms: self
6422                        .daemon_provenance
6423                        .start_clock
6424                        .map(|clock| clock.started_at_ms())
6425                        .or(self.daemon_provenance.started_at_ms),
6426                    running_image: self
6427                        .daemon_provenance
6428                        .probe
6429                        .observe(
6430                            self.daemon_provenance.pid,
6431                            self.daemon_provenance.executable_path.as_deref(),
6432                            self.daemon_provenance.executable_identity.clone(),
6433                            self.daemon_provenance.process_start_time,
6434                        )
6435                        .await,
6436                },
6437            };
6438            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6439            Ok(vec![control_response_body_frame(
6440                &frame,
6441                &response,
6442                "ClientControlResponse::SupervisorProvenance",
6443            )?])
6444        })
6445        .await
6446    }
6447
6448    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6449        self.refresh_capability_requirements();
6450        let generation = self
6451            .registry
6452            .generation()
6453            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6454        let modules = self
6455            .supervisor
6456            .list()
6457            .into_iter()
6458            .map(|module| {
6459                let status = module.status_for_control("health").map_err(|err| {
6460                    RouterError::backend(
6461                        0,
6462                        frame.header.corr,
6463                        format!("failed to read supervisor health: {err}"),
6464                    )
6465                })?;
6466                let module_id = status.module_id;
6467                let capability_detail = self
6468                    .capability_evaluator
6469                    .required_problem_detail(&module_id);
6470                Ok(SupervisorHealthEntry {
6471                    module_id,
6472                    status: status.health.status,
6473                    detail: append_capability_problem_detail(
6474                        status.health.detail,
6475                        capability_detail,
6476                    ),
6477                    metrics: status.health.metrics,
6478                    consecutive_failures: status.health.consecutive_failures,
6479                    late_answer_count: status.health.late_answer_count,
6480                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6481                    last_action: status.health.last_action,
6482                    last_action_ms: status.health.last_action_ms,
6483                    last_probe_ms: status.health.last_probe_ms,
6484                })
6485            })
6486            .collect::<Result<Vec<_>, RouterError>>()?;
6487        let response = ClientControlResponse::SupervisorHealth {
6488            generation,
6489            modules,
6490        };
6491        Ok(vec![control_response_body_frame(
6492            &frame,
6493            &response,
6494            "ClientControlResponse::SupervisorHealth",
6495        )?])
6496    }
6497
6498    async fn handle_supervisor_restart(
6499        &self,
6500        frame: Frame,
6501        module_id: String,
6502        drain_timeout_ms: Option<u64>,
6503        requester: Principal,
6504    ) -> Result<Vec<Frame>, RouterError> {
6505        let operation_lock = self.supervisor.operation_lock();
6506        let _operation_guard = operation_lock.lock().await;
6507        let Some(module) = self.supervisor.get(&module_id) else {
6508            return Ok(vec![control_error_frame(
6509                &frame,
6510                "unknown_module",
6511                format!("module_id '{module_id}' is not supervised"),
6512            )?]);
6513        };
6514
6515        self.route_outages.mark_operator_action(&module_id);
6516        if let Err(err) = module
6517            .restart_requested(drain_timeout_ms, Some(requester))
6518            .await
6519        {
6520            self.route_outages
6521                .operator_action_ended_unrefused(&module_id);
6522            let (code, message) = match err {
6523                crate::supervise::SuperviseError::Disabled { .. } => {
6524                    ("module_disabled", err.to_string())
6525                }
6526                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6527                    ("swap_in_progress", err.to_string())
6528                }
6529                _ => (
6530                    "target_unavailable",
6531                    format!("failed to restart module_id '{module_id}': {err}"),
6532                ),
6533            };
6534            return Ok(vec![control_error_frame(&frame, code, message)?]);
6535        }
6536
6537        let response = ClientControlResponse::SupervisorAck {
6538            module_id,
6539            applied: true,
6540        };
6541        Ok(vec![control_response_body_frame(
6542            &frame,
6543            &response,
6544            "ClientControlResponse::SupervisorAck",
6545        )?])
6546    }
6547
6548    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
6549    /// when the old process has finished draining: a caller whose own lane
6550    /// rides the old process must get its reply before that drain waits on it.
6551    async fn handle_supervisor_swap(
6552        &self,
6553        frame: Frame,
6554        module_id: String,
6555        ready_timeout_ms: Option<u64>,
6556        requester: Principal,
6557    ) -> Result<Vec<Frame>, RouterError> {
6558        // The daemon-wide operation lock is held only to resolve the handle,
6559        // not across the swap. The swap can take its whole readiness budget,
6560        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6561        // holding it here would park an operator's stop behind the swap it is
6562        // meant to abort. A rescan or stop that reaches the module during the
6563        // swap is served by the swap itself (see `supervise_swap`).
6564        let module = {
6565            let operation_lock = self.supervisor.operation_lock();
6566            let _operation_guard = operation_lock.lock().await;
6567            self.supervisor.get(&module_id)
6568        };
6569        let Some(module) = module else {
6570            return Ok(vec![control_error_frame(
6571                &frame,
6572                "unknown_module",
6573                format!("module_id '{module_id}' is not supervised"),
6574            )?]);
6575        };
6576
6577        self.route_outages.mark_operator_action(&module_id);
6578        if let Err(err) = module
6579            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6580            .await
6581        {
6582            self.route_outages
6583                .operator_action_ended_unrefused(&module_id);
6584            use crate::supervise::SuperviseError;
6585            let message = err.to_string();
6586            let error = match err {
6587                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6588                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6589                    code: "swap_refused".to_string(),
6590                    message,
6591                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6592                },
6593                SuperviseError::SwapFailed {
6594                    arm,
6595                    candidate_exit,
6596                    ..
6597                } => ErrorBody {
6598                    code: "swap_failed".to_string(),
6599                    message,
6600                    detail: Some(serde_json::json!({
6601                        "arm": arm.as_str(),
6602                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6603                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6604                    })),
6605                },
6606                _ => ErrorBody::new(
6607                    "target_unavailable",
6608                    format!("failed to swap module_id '{module_id}': {message}"),
6609                ),
6610            };
6611            return Ok(vec![control_error_body_frame(&frame, error)?]);
6612        }
6613        // A completed swap kept the incumbent serving until cutover, so it
6614        // usually opened no outage; a mark left behind would make the next,
6615        // unrelated outage read as requested.
6616        self.route_outages
6617            .operator_action_ended_unrefused(&module_id);
6618
6619        let response = ClientControlResponse::SupervisorAck {
6620            module_id,
6621            applied: true,
6622        };
6623        Ok(vec![control_response_body_frame(
6624            &frame,
6625            &response,
6626            "ClientControlResponse::SupervisorAck",
6627        )?])
6628    }
6629
6630    async fn handle_supervisor_reload(
6631        &self,
6632        frame: Frame,
6633        module_id: String,
6634        requester: Principal,
6635    ) -> Result<Vec<Frame>, RouterError> {
6636        let operation_lock = self.supervisor.operation_lock();
6637        let _operation_guard = operation_lock.lock().await;
6638        let Some(module) = self.supervisor.get(&module_id) else {
6639            return Ok(vec![control_error_frame(
6640                &frame,
6641                "unknown_module",
6642                format!("module_id '{module_id}' is not supervised"),
6643            )?]);
6644        };
6645
6646        self.route_outages.mark_operator_action(&module_id);
6647        if let Err(err) = module.reload_requested(Some(requester)).await {
6648            self.route_outages
6649                .operator_action_ended_unrefused(&module_id);
6650            let (code, message) = match err {
6651                crate::supervise::SuperviseError::Disabled { .. } => {
6652                    ("module_disabled", err.to_string())
6653                }
6654                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6655                    ("swap_in_progress", err.to_string())
6656                }
6657                _ => (
6658                    "reload_failed",
6659                    format!("failed to reload module_id '{module_id}': {err}"),
6660                ),
6661            };
6662            return Ok(vec![control_error_frame(&frame, code, message)?]);
6663        }
6664
6665        let response = ClientControlResponse::SupervisorAck {
6666            module_id,
6667            applied: true,
6668        };
6669        Ok(vec![control_response_body_frame(
6670            &frame,
6671            &response,
6672            "ClientControlResponse::SupervisorAck",
6673        )?])
6674    }
6675
6676    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6677    /// the configuration comes from and whether to apply it; every mode then
6678    /// runs the same comparison, so a preview describes the apply it previews.
6679    async fn handle_supervisor_rescan(
6680        &self,
6681        frame: Frame,
6682        mode: RescanMode,
6683        requester: Principal,
6684    ) -> Result<Vec<Frame>, RouterError> {
6685        let preview = mode.previews();
6686        let Some(context) = self.rescan.clone() else {
6687            return Ok(vec![control_error_frame(
6688                &frame,
6689                "rescan_unavailable",
6690                "the daemon was not started with a reloadable config path".to_string(),
6691            )?]);
6692        };
6693
6694        let operation_lock = self.supervisor.operation_lock();
6695        let _operation_guard = operation_lock.lock().await;
6696        // Loaded under the operation lock, so no other rescan can change the
6697        // staged configuration between a checked apply's comparison and the
6698        // apply itself.
6699        let loaded = match self.load_rescan_config(&mode, &context) {
6700            Ok(config) => config,
6701            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6702        };
6703        // `load` reports a missing file as Ok(None), which is correct at boot
6704        // (no config, nothing to supervise) and catastrophic here: rescan treats
6705        // "not in the config" as "remove it", so an absent file would read as an
6706        // empty module list and retire the entire running fleet. An editor
6707        // writing via write-new-then-rename, or a half-finished edit, is enough
6708        // to open that window. Refuse instead: a config that cannot be read
6709        // carries no instruction to remove anything.
6710        let Some((config, activity_config)) = loaded else {
6711            return Ok(vec![control_error_frame(
6712                &frame,
6713                "invalid_daemon_config",
6714                format!(
6715                    "daemon config not found at {}; refusing to rescan (an absent config would \
6716                     retire every supervised module)",
6717                    context.config_path.display()
6718                ),
6719            )?]);
6720        };
6721        let (
6722            configured_port,
6723            storage_config,
6724            admission_facts_carrier_module_id,
6725            admission_facts_targets,
6726            scope_authority_owners,
6727            federated_scope_owner,
6728            privileged_operator_prompt,
6729            modules,
6730            reserved_capabilities,
6731        ) = (
6732            config.port,
6733            config.storage,
6734            config.admission_facts_carrier_module_id,
6735            config.admission_facts_targets,
6736            config.scope_authority_owners,
6737            config.federated_scope_owner,
6738            config.privileged_operator_prompt,
6739            config.modules,
6740            config.reserved_capabilities,
6741        );
6742
6743        // Collect the sections rescan cannot apply, so the REPLY carries them.
6744        //
6745        // The warning below has always been correct and has always gone only to
6746        // the journal -- addressed to whoever reads logs, while the person who
6747        // just edited the config is looking at the CLI. Naming each section
6748        // individually rather than setting a flag: "something outside modules
6749        // changed" sends the operator back to diffing their own file, which is
6750        // the work this is meant to save.
6751        let mut restart_required = Vec::new();
6752        for section in RestartRequiredSection::ALL {
6753            let changed = match section {
6754                RestartRequiredSection::Port => configured_port != context.configured_port,
6755                RestartRequiredSection::Storage => storage_config != context.storage_config,
6756                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6757                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6758                }
6759                RestartRequiredSection::AdmissionFactsTargets => {
6760                    admission_facts_targets != context.admission_facts_targets
6761                }
6762                RestartRequiredSection::ScopeAuthorityOwners => {
6763                    scope_authority_owners != context.scope_authority_owners
6764                }
6765                RestartRequiredSection::FederatedScopeOwner => {
6766                    federated_scope_owner != context.federated_scope_owner
6767                }
6768                RestartRequiredSection::PrivilegedOperatorPrompt => {
6769                    privileged_operator_prompt != context.privileged_operator_prompt
6770                }
6771            };
6772            if changed {
6773                restart_required.push(section.label().to_string());
6774            }
6775        }
6776        if !restart_required.is_empty() {
6777            warn!(
6778                config_path = %context.config_path.display(),
6779                sections = %restart_required.join(", "),
6780                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6781            );
6782        }
6783
6784        for configured in &modules {
6785            if let Err(err) = validate_spec(&configured.module_spec()) {
6786                return Ok(vec![control_error_frame(
6787                    &frame,
6788                    "invalid_daemon_config",
6789                    format!("supervisor rescan rejected daemon config: {err}"),
6790                )?]);
6791            }
6792        }
6793
6794        let configured_capabilities = modules
6795            .iter()
6796            .map(|module| (module.module_id.clone(), module.enabled))
6797            .collect::<Vec<_>>();
6798        let preview_capability_warnings = if preview {
6799            let (_, registrations) = self.runtime_capability_snapshot()?;
6800            let current_modules = self
6801                .supervisor
6802                .list()
6803                .into_iter()
6804                .map(|module| module.module_id().to_string())
6805                .collect::<BTreeSet<_>>();
6806            let resulting_modules = configured_capabilities.clone();
6807            let removed = current_modules
6808                .into_iter()
6809                .filter(|module_id| {
6810                    !resulting_modules
6811                        .iter()
6812                        .any(|(configured_id, _)| configured_id == module_id)
6813                })
6814                .collect::<Vec<_>>();
6815            // A candidate preview also names providers the candidate disables.
6816            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6817            let disabled = match &mode {
6818                RescanMode::Candidate(_) => resulting_modules
6819                    .iter()
6820                    .filter(|(_, enabled)| !enabled)
6821                    .map(|(module_id, _)| module_id.clone())
6822                    .collect::<Vec<_>>(),
6823                _ => Vec::new(),
6824            };
6825            self.capability_evaluator.preview_warnings(
6826                resulting_modules,
6827                &removed,
6828                &disabled,
6829                &registrations,
6830            )
6831        } else {
6832            Vec::new()
6833        };
6834        let reconciled = match &mode {
6835            RescanMode::Candidate(candidate) => self
6836                .preview_candidate(&context, modules, candidate)
6837                .map(|result| (result, Vec::new())),
6838            _ => {
6839                self.reconcile_supervised_modules(
6840                    &context.supervisor,
6841                    modules,
6842                    preview,
6843                    requester.clone(),
6844                )
6845                .await
6846            }
6847        };
6848        let (result, changed) = match reconciled {
6849            Ok(result) => result,
6850            Err(message) => {
6851                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6852            }
6853        };
6854        if !preview {
6855            if let Some(ledger) = &self.activity {
6856                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6857                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6858            }
6859            if let Err(code) = self.configure_activity(activity_config).await {
6860                warn!(
6861                    code,
6862                    "activity retention could not be applied during rescan"
6863                );
6864            }
6865            self.capability_evaluator
6866                .configure(configured_capabilities, reserved_capabilities);
6867            self.capability_evaluator.wake_deadline_loop();
6868            self.refresh_capability_requirements();
6869        }
6870        let mut result = result;
6871        result.restart_required = restart_required;
6872        result.capability_warnings = preview_capability_warnings;
6873        let response = ClientControlResponse::SupervisorRescan { result };
6874        Ok(vec![control_response_body_frame(
6875            &frame,
6876            &response,
6877            "ClientControlResponse::SupervisorRescan",
6878        )?])
6879    }
6880
6881    async fn handle_supervisor_release_reserved(
6882        &self,
6883        frame: Frame,
6884        module_id: String,
6885    ) -> Result<Vec<Frame>, RouterError> {
6886        let Some(context) = self.rescan.clone() else {
6887            return Ok(vec![control_error_frame(
6888                &frame,
6889                "release_unavailable",
6890                "reserved-id release requires a daemon started with a reloadable config path",
6891            )?]);
6892        };
6893        let operation_lock = self.supervisor.operation_lock();
6894        let _operation_guard = operation_lock.lock().await;
6895        let loaded = match crate::daemon_config::load(&context.config_path) {
6896            Ok(Some(config)) => config,
6897            Ok(None) => {
6898                return Ok(vec![control_error_frame(
6899                    &frame,
6900                    "invalid_daemon_config",
6901                    format!(
6902                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6903                        context.config_path.display()
6904                    ),
6905                )?])
6906            }
6907            Err(err) => {
6908                return Ok(vec![control_error_frame(
6909                    &frame,
6910                    "invalid_daemon_config",
6911                    format!("unable to verify reserved-id release against daemon config: {err}"),
6912                )?])
6913            }
6914        };
6915        if loaded
6916            .modules
6917            .iter()
6918            .any(|configured| configured.module_id == module_id)
6919        {
6920            return Ok(vec![control_error_frame(
6921                &frame,
6922                "reserved_module_configured",
6923                format!(
6924                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6925                ),
6926            )?]);
6927        }
6928        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6929            return Ok(vec![control_error_frame(
6930                &frame,
6931                "reserved_gate_not_retained",
6932                format!(
6933                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6934                ),
6935            )?]);
6936        }
6937
6938        let response = ClientControlResponse::SupervisorAck {
6939            module_id,
6940            applied: true,
6941        };
6942        Ok(vec![control_response_body_frame(
6943            &frame,
6944            &response,
6945            "ClientControlResponse::SupervisorAck",
6946        )?])
6947    }
6948
6949    /// Compare the configured module set with the supervised one, changing
6950    /// nothing. The staged spec each supervised module holds is what the
6951    /// configured entry is compared with.
6952    fn plan_rescan(
6953        &self,
6954        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6955    ) -> Result<RescanPlan, String> {
6956        let mut current = BTreeMap::new();
6957        for module in self.supervisor.list() {
6958            let (spec, health) = module.configuration().map_err(|err| {
6959                format!(
6960                    "failed to read configuration for module_id '{}': {err}",
6961                    module.module_id()
6962                )
6963            })?;
6964            let enabled = module
6965                .status()
6966                .map_err(|err| {
6967                    format!(
6968                        "failed to read status for module_id '{}': {err}",
6969                        module.module_id()
6970                    )
6971                })?
6972                .enabled;
6973            current.insert(
6974                module.module_id().to_string(),
6975                (module, spec, health, enabled),
6976            );
6977        }
6978        let configured = configured_modules
6979            .into_iter()
6980            .map(|module| (module.module_id.clone(), module))
6981            .collect::<BTreeMap<_, _>>();
6982
6983        let added = configured
6984            .keys()
6985            .filter(|module_id| !current.contains_key(*module_id))
6986            .cloned()
6987            .collect::<Vec<_>>();
6988        let removed = current
6989            .keys()
6990            .filter(|module_id| !configured.contains_key(*module_id))
6991            .cloned()
6992            .collect::<Vec<_>>();
6993        let mut changed_pending_reload = Vec::new();
6994        let mut configuration_changes = BTreeSet::new();
6995        let mut enabled_changes = BTreeSet::new();
6996        let mut unchanged = 0_u32;
6997
6998        for (module_id, configured_module) in &configured {
6999            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
7000            else {
7001                continue;
7002            };
7003            // Compare the whole launch spec so a future launch field cannot
7004            // accidentally become a live-only policy change. Health is stored
7005            // separately and applies live without replacing the process.
7006            let launch_changed = *current_spec != configured_module.module_spec();
7007            let configuration_changed =
7008                launch_changed || *current_health != configured_module.health;
7009            // Applying a rescan also replaces the recorded config value, even
7010            // when an operator already put the runtime in the requested state.
7011            let enabled_changed = *current_enabled != configured_module.enabled
7012                || self
7013                    .capability_evaluator
7014                    .configured_enabled(module_id)
7015                    .is_some_and(|enabled| enabled != configured_module.enabled);
7016            if configuration_changed {
7017                configuration_changes.insert(module_id.clone());
7018            }
7019            if launch_changed {
7020                changed_pending_reload.push(module_id.clone());
7021            }
7022            if enabled_changed {
7023                enabled_changes.insert(module_id.clone());
7024            }
7025            if !configuration_changed && !enabled_changed {
7026                unchanged = unchanged.saturating_add(1);
7027            }
7028        }
7029        Ok(RescanPlan {
7030            current,
7031            configured,
7032            added,
7033            removed,
7034            changed_pending_reload,
7035            configuration_changes,
7036            enabled_changes,
7037            unchanged,
7038        })
7039    }
7040
7041    /// Reconcile the running module set against the configured one.
7042    ///
7043    /// With `preview` set, the diff is computed and returned WITHOUT applying any
7044    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
7045    /// deliberately shares this function with the executing path rather than
7046    /// computing the same diff somewhere else -- two implementations of one
7047    /// decision agree until they do not, and the whole value of a preview is that
7048    /// it describes the operation that will actually run.
7049    async fn reconcile_supervised_modules(
7050        &self,
7051        supervisor: &Supervisor,
7052        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7053        preview: bool,
7054        requester: Principal,
7055    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7056        let plan = self.plan_rescan(configured_modules)?;
7057
7058        // Everything above this point is pure computation over two snapshots.
7059        // Everything below MUTATES. The preview returns here so the boundary is a
7060        // single early return rather than a condition repeated at each mutation
7061        // site, where one missed guard would apply part of a change the caller was
7062        // told would not happen.
7063        if preview {
7064            return Ok((plan.preview_result(), Vec::new()));
7065        }
7066        let changed = plan
7067            .configuration_changes
7068            .union(&plan.enabled_changes)
7069            .cloned()
7070            .collect::<Vec<_>>();
7071        let RescanPlan {
7072            current,
7073            configured,
7074            added,
7075            removed,
7076            changed_pending_reload,
7077            configuration_changes,
7078            enabled_changes,
7079            unchanged,
7080        } = plan;
7081        // The staged generation moves before the first change rather than
7082        // after the last, so a rescan that fails partway still invalidates
7083        // every earlier `supervisor.rescan_checked` preview: part of its diff
7084        // may have run.
7085        if !(added.is_empty()
7086            && removed.is_empty()
7087            && configuration_changes.is_empty()
7088            && enabled_changes.is_empty())
7089        {
7090            if let Some(context) = &self.rescan {
7091                context
7092                    .generation
7093                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7094            }
7095        }
7096
7097        for module_id in &removed {
7098            let module = &current
7099                .get(module_id)
7100                .expect("removed module came from current supervisor state")
7101                .0;
7102            module
7103                .retire_requested(Some(requester.clone()))
7104                .await
7105                .map_err(|err| {
7106                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
7107                })?;
7108            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
7109            //
7110            // `handle_route_open` resolves an absent module in three steps:
7111            // registry, then supervisor status, then tombstone. Retiring first
7112            // opens a window where ALL THREE ARE ABSENT -- the registry entry
7113            // went with the teardown above, the supervisor entry went with
7114            // `retire`, and the tombstone does not exist yet -- so a route.open
7115            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
7116            // it") for a module that was deliberately removed and whose caller
7117            // should get `module_removed` (TERMINAL, carrying a removal age).
7118            //
7119            // Writing the tombstone first closes it: during the window the
7120            // supervisor entry still answers, so the caller gets
7121            // `target_unavailable` -- retryable, and TRUE, because the module
7122            // is mid-teardown. After both statements it is `module_removed`.
7123            // No instant remains where a removed module reads as one that
7124            // never existed.
7125            //
7126            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
7127            // the absence of a test beside a fix invites deletion: these are
7128            // two sync statements with no await between them, so reaching the
7129            // window needs a second worker thread to land exactly between them
7130            // and there is no hook to force it. MEASURED: the 25 daemon_config
7131            // tests pass identically with the old order and the new one, so
7132            // the existing suite cannot see this and a green run is not
7133            // evidence either way. What the suite does hold is the
7134            // post-condition -- a removed module answers `module_removed` --
7135            // which this preserves.
7136            //
7137            // Found by an Athena panel reading the shipped tree against a
7138            // design note (2026-09-19), as the one concrete instance of that
7139            // note's class that survived contact with source. Direction is
7140            // benign: retryable where terminal was intended, never the reverse.
7141            self.supervisor.record_rescan_removal(module_id);
7142            self.supervisor.retire(module_id);
7143            self.route_outages.forget(module_id);
7144        }
7145
7146        for module_id in configured.keys() {
7147            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7148                continue;
7149            };
7150            let configured_module = configured
7151                .get(module_id)
7152                .expect("configured module id came from configured map");
7153            if configuration_changes.contains(module_id) {
7154                module
7155                    .update_configuration(
7156                        configured_module.module_spec(),
7157                        configured_module.health.clone(),
7158                        configured_module.drain_timeout_ms,
7159                    )
7160                    .await
7161                    .map_err(|err| {
7162                        format!(
7163                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
7164                        )
7165                    })?;
7166            }
7167            // Config-only drift is applied by the evaluator's config update;
7168            // do not turn it into a runtime start or stop.
7169            if *current_enabled != configured_module.enabled {
7170                // A rescan that starts or stops a module applies an operator's
7171                // edit to the config, so the resulting outage was asked for.
7172                self.route_outages.mark_operator_action(module_id);
7173                module
7174                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7175                    .await
7176                    .map_err(|err| {
7177                        self.route_outages.operator_action_ended_unrefused(module_id);
7178                        format!(
7179                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7180                            configured_module.enabled
7181                        )
7182                    })?;
7183            }
7184        }
7185
7186        for module_id in &added {
7187            let configured_module = configured
7188                .get(module_id)
7189                .expect("added module id came from configured map");
7190            supervisor
7191                .supervise_configured_requested(
7192                    configured_module.module_spec(),
7193                    configured_module.enabled,
7194                    configured_module.health.clone(),
7195                    configured_module.drain_timeout_ms,
7196                    configured_module.restart,
7197                    configured_module.register_timeout,
7198                    Some(requester.clone()),
7199                )
7200                .map_err(|err| {
7201                    format!("failed to add module_id '{module_id}' during rescan: {err}")
7202                })?;
7203        }
7204
7205        Ok((
7206            SupervisorRescanResult {
7207                added,
7208                removed,
7209                changed_pending_reload,
7210                enabled_changes: enabled_changes.iter().cloned().collect(),
7211                unchanged,
7212                preview: false,
7213                // Filled by the caller, which is the only layer that can see the
7214                // previous config to diff against.
7215                restart_required: Vec::new(),
7216                capability_warnings: Vec::new(),
7217                module_actions: Vec::new(),
7218                earlier_pending: Vec::new(),
7219                expect: None,
7220            },
7221            changed,
7222        ))
7223    }
7224
7225    async fn handle_supervisor_set_enabled(
7226        &self,
7227        frame: Frame,
7228        module_id: String,
7229        enabled: bool,
7230        requester: Principal,
7231    ) -> Result<Vec<Frame>, RouterError> {
7232        let operation_lock = self.supervisor.operation_lock();
7233        let _operation_guard = operation_lock.lock().await;
7234        let Some(module) = self.supervisor.get(&module_id) else {
7235            return Ok(vec![control_error_frame(
7236                &frame,
7237                "unknown_module",
7238                format!("module_id '{module_id}' is not supervised"),
7239            )?]);
7240        };
7241
7242        // Enabling counts as well as disabling: a module an operator starts
7243        // is refused until it registers, and that wait was asked for.
7244        self.route_outages.mark_operator_action(&module_id);
7245        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7246            Ok(applied) => applied,
7247            Err(err) => {
7248                self.route_outages
7249                    .operator_action_ended_unrefused(&module_id);
7250                return Ok(vec![control_error_frame(
7251                    &frame,
7252                    "target_unavailable",
7253                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7254                )?]);
7255            }
7256        };
7257        if !applied {
7258            // Already in the requested state: nothing was made unavailable,
7259            // so the mark must not outlive this request.
7260            self.route_outages
7261                .operator_action_ended_unrefused(&module_id);
7262        }
7263
7264        self.capability_evaluator.wake_deadline_loop();
7265        self.refresh_capability_requirements();
7266        let response = ClientControlResponse::SupervisorAck { module_id, applied };
7267        Ok(vec![control_response_body_frame(
7268            &frame,
7269            &response,
7270            "ClientControlResponse::SupervisorAck",
7271        )?])
7272    }
7273
7274    async fn handle_supervisor_health_probe(
7275        &self,
7276        ctx: &RouteCtx,
7277        frame: Frame,
7278        module_id: String,
7279    ) -> Result<Vec<Frame>, RouterError> {
7280        self.refresh_capability_requirements();
7281        let Some(registration) = self
7282            .registry
7283            .get_module(&module_id)
7284            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7285        else {
7286            return Ok(vec![control_error_frame(
7287                &frame,
7288                "unknown_module",
7289                format!("module_id '{module_id}' is not registered"),
7290            )?]);
7291        };
7292
7293        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
7294        // named for it. Making `module_registration_grants_op` return false
7295        // unconditionally reddens five tests, and every one is named for something
7296        // else -- capability relay, probe/bind demultiplexing, supervision-only
7297        // probing. They exercise a successful advertisement check on the way to their
7298        // own subject.
7299        //
7300        // Real protection, fragile in a specific way: narrowing any of those tests to
7301        // focus on its stated subject would silently remove coverage nobody knows
7302        // they are carrying. Recorded here rather than as a sixth test, because the
7303        // useful fact is WHICH tests hold the guard up -- a new test would add
7304        // coverage without telling the next person what the existing ones quietly do.
7305        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7306        {
7307            return Ok(vec![control_error_frame(
7308                &frame,
7309                "health_not_advertised",
7310                format!("module_id '{module_id}' did not advertise health.check"),
7311            )?]);
7312        }
7313
7314        self.observe_control_wait(
7315            ctx,
7316            Some(&registration.manifest.module_id),
7317            ops::SUPERVISOR_HEALTH_PROBE,
7318            async {
7319                let deadline = Instant::now() + self.health_probe_timeout;
7320                let pending = match self.forwarding.begin_module_control_rpc_for(
7321                    &module_id,
7322                    MODULE_CONTROL_OP_HEALTH_CHECK,
7323                    deadline,
7324                ) {
7325                    Ok(pending) => pending,
7326                    Err(err) => {
7327                        return Ok(vec![control_error_frame(
7328                            &frame,
7329                            forwarding_error_code(&err),
7330                            err.to_string(),
7331                        )?])
7332                    }
7333                };
7334
7335                let PendingModuleControlRpc {
7336                    endpoint,
7337                    module_sink,
7338                    negotiated_ver,
7339                    corr: probe_corr,
7340                    receiver,
7341                } = pending;
7342                let mut guard =
7343                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7344                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7345                    .map_err(|err| {
7346                        RouterError::backend(
7347                            0,
7348                            frame.header.corr,
7349                            format!("failed to encode health.check request: {err}"),
7350                        )
7351                    })?;
7352                let probe_frame = Frame::build_with_version(
7353                    negotiated_ver,
7354                    FrameType::Request,
7355                    control_flags(),
7356                    0,
7357                    0,
7358                    probe_corr,
7359                    probe_body,
7360                )
7361                .map_err(RouterError::FrameBuild)?;
7362
7363                // Queue admission and the reply share one deadline, so a module that
7364                // stops reading cannot park the requesting connection's reader forever.
7365                match timeout_at(deadline, module_sink.send(probe_frame)).await {
7366                    Ok(Ok(())) => {}
7367                    Ok(Err(err)) => {
7368                        return Ok(vec![control_error_frame(
7369                            &frame,
7370                            "target_unavailable",
7371                            err.to_string(),
7372                        )?]);
7373                    }
7374                    Err(_) => {
7375                        return Ok(vec![control_error_frame(
7376                            &frame,
7377                            "module_timeout",
7378                            format!(
7379                                "module_id '{module_id}' did not answer health.check within {:?}",
7380                                self.health_probe_timeout
7381                            ),
7382                        )?]);
7383                    }
7384                }
7385
7386                match timeout_at(deadline, receiver).await {
7387                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7388                        guard.disarm();
7389                        let Some(report) = response.health_report() else {
7390                            return Ok(vec![control_error_frame(
7391                                &frame,
7392                                "invalid_control_body",
7393                                "health.check RPC returned a non-health response",
7394                            )?]);
7395                        };
7396                        // Metrics go out whole here. The supervisor's cached snapshot
7397                        // caps this blob (see truncate_health_metrics), and this path
7398                        // exists precisely to answer without that cap -- so applying it
7399                        // here would leave no way to see what the cached view drops.
7400                        let HealthReport {
7401                            status,
7402                            detail,
7403                            metrics,
7404                        } = report;
7405                        let capability_detail = self
7406                            .capability_evaluator
7407                            .required_problem_detail(&module_id);
7408                        let response = ClientControlResponse::SupervisorHealthProbe {
7409                            module_id,
7410                            status,
7411                            detail: append_capability_problem_detail(detail, capability_detail),
7412                            metrics,
7413                        };
7414                        Ok(vec![control_response_body_frame(
7415                            &frame,
7416                            &response,
7417                            "ClientControlResponse::SupervisorHealthProbe",
7418                        )?])
7419                    }
7420                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7421                        guard.disarm();
7422                        // The module refused the health probe with its own body, so
7423                        // the refusal is relayed without the daemon-origin flag
7424                        // (bit 6): a client must not read it as the daemon's own
7425                        // refusal, which it may treat as never forwarded.
7426                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7427                    }
7428                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7429                        guard.disarm();
7430                        Ok(vec![control_error_frame(
7431                            &frame,
7432                            "target_unavailable",
7433                            message,
7434                        )?])
7435                    }
7436                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7437                        guard.disarm();
7438                        Ok(vec![control_error_frame(
7439                            &frame,
7440                            "invalid_control_body",
7441                            message,
7442                        )?])
7443                    }
7444                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7445                        guard.disarm();
7446                        Ok(vec![control_error_frame(
7447                            &frame,
7448                            "invalid_control_body",
7449                            format!("expected module-control op '{expected}', got '{actual}'"),
7450                        )?])
7451                    }
7452                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7453                        guard.disarm();
7454                        Ok(vec![control_error_frame(
7455                            &frame,
7456                            "module_timeout",
7457                            format!(
7458                                "module_id '{module_id}' answered health.check after {:?}",
7459                                self.health_probe_timeout
7460                            ),
7461                        )?])
7462                    }
7463                    Ok(Err(_)) => Ok(vec![control_error_frame(
7464                        &frame,
7465                        "target_unavailable",
7466                        "health.check waiter was canceled before the module responded",
7467                    )?]),
7468                    Err(_) => Ok(vec![control_error_frame(
7469                        &frame,
7470                        "module_timeout",
7471                        format!(
7472                            "module_id '{module_id}' did not answer health.check within {:?}",
7473                            self.health_probe_timeout
7474                        ),
7475                    )?]),
7476                }
7477            },
7478        )
7479        .await
7480    }
7481
7482    fn supervisor_status(
7483        &self,
7484        module_id: &str,
7485        corr: u64,
7486    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7487        self.supervisor
7488            .get(module_id)
7489            .map(|module| {
7490                let warming = module.is_warming_for_control("status").map_err(|err| {
7491                    RouterError::backend(
7492                        0,
7493                        corr,
7494                        format!(
7495                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
7496                        ),
7497                    )
7498                })?;
7499                module.status_for_control("status").map_err(|err| {
7500                    RouterError::backend(
7501                        0,
7502                        corr,
7503                        format!(
7504                            "failed to read supervisor status for module_id '{module_id}': {err}"
7505                        ),
7506                    )
7507                }).map(|status| (status, warming))
7508            })
7509            .transpose()
7510    }
7511
7512    fn guard_module_control_op(
7513        &self,
7514        frame: &Frame,
7515        module_id: &str,
7516        op: &str,
7517    ) -> Result<Option<Frame>, RouterError> {
7518        if self.module_grants_op(module_id, op, frame.header.corr)? {
7519            return Ok(None);
7520        }
7521
7522        Ok(Some(control_error_frame(
7523            frame,
7524            "op_not_allowed",
7525            format!("module_id '{module_id}' did not grant control op '{op}'"),
7526        )?))
7527    }
7528
7529    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7530        let Some(registration) = self
7531            .registry
7532            .get_module(module_id)
7533            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7534        else {
7535            return Ok(false);
7536        };
7537        Ok(module_registration_grants_op(&registration.control_ops, op))
7538    }
7539
7540    fn handle_status_update(
7541        &self,
7542        endpoint: ModuleEndpointId,
7543        frame: Frame,
7544    ) -> Result<Vec<Frame>, RouterError> {
7545        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7546            Ok(update) => update,
7547            Err(err) => {
7548                // Forward-compat: a newer module may push a channel-0 op this subc
7549                // version doesn't know. The control contract says unknown push ops
7550                // are IGNORED, never answered with an error. Only a malformed body
7551                // for an op we DO know is a real error worth surfacing.
7552                if is_known_module_push_op(&frame.body) {
7553                    return Ok(vec![control_error_frame(
7554                        &frame,
7555                        "invalid_control_body",
7556                        format!("malformed module control push body: {err}"),
7557                    )?]);
7558                }
7559                return Ok(Vec::new());
7560            }
7561        };
7562
7563        match update {
7564            ModuleControlPush::RouteStatus {
7565                route_channel,
7566                route_epoch,
7567                status,
7568            } => {
7569                self.forwarding
7570                    .cache_status(endpoint, route_channel, route_epoch, status)
7571                    .map_err(RouterError::Forwarding)?;
7572            }
7573        }
7574        Ok(Vec::new())
7575    }
7576
7577    fn handle_route_poll(
7578        &self,
7579        ctx: &RouteCtx,
7580        frame: Frame,
7581        route_channel: u16,
7582        route_epoch: u32,
7583        kind: PollKind,
7584    ) -> Result<Vec<Frame>, RouterError> {
7585        let snapshot = self
7586            .forwarding
7587            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7588            .map_err(RouterError::Forwarding)?;
7589        let response = match (kind, snapshot) {
7590            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7591                ClientControlResponse::RoutePoll {
7592                    route_channel,
7593                    route_epoch,
7594                    status,
7595                    live: None,
7596                }
7597            }
7598            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7599                route_channel,
7600                route_epoch,
7601                status: None,
7602                live: None,
7603            },
7604            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7605                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7606                // is what makes reporting `true` correct rather than a
7607                // confident guess. `process_live` returns None only when the
7608                // module id has no supervisor snapshot at all -- an
7609                // externally-started module the daemon did not spawn -- and
7610                // for those the supervisor has no opinion to offer, ever. It
7611                // is never None for a supervised module in an unknown state:
7612                // a supervised module always has a snapshot, and the answer
7613                // comes from `state == Running && process_alive`.
7614                //
7615                // The route is Bound, so the module completed a HELLO on a
7616                // live connection; "the process this route points at is
7617                // running" is therefore attested by the binding rather than
7618                // assumed. Reporting `false` for an unsupervised module would
7619                // be the actual lie -- it would tell a client its healthy
7620                // route is dead because the daemon does not manage the
7621                // process.
7622                //
7623                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7624                // module whose liveness is genuinely unknown, e.g. a snapshot
7625                // that has not been populated yet -- THIS DEFAULT BECOMES
7626                // WRONG and must split: unsupervised stays true, unknown
7627                // becomes null so the client can tell the two apart. The
7628                // response field is already `Option<bool>`, so the wire can
7629                // carry that distinction today.
7630                let live = self
7631                    .process_liveness
7632                    .as_ref()
7633                    .and_then(|source| source.process_live(&module_id))
7634                    .unwrap_or(true);
7635                ClientControlResponse::RoutePoll {
7636                    route_channel,
7637                    route_epoch,
7638                    status: None,
7639                    live: Some(live),
7640                }
7641            }
7642            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7643                route_channel,
7644                route_epoch,
7645                status: None,
7646                live: Some(false),
7647            },
7648        };
7649
7650        Ok(vec![control_response_body_frame(
7651            &frame,
7652            &response,
7653            "ClientControlResponse::RoutePoll",
7654        )?])
7655    }
7656
7657    pub(crate) fn observe_module_control_completion(
7658        &self,
7659        completion: ModuleControlRpcCompletion,
7660    ) -> bool {
7661        match completion {
7662            ModuleControlRpcCompletion::Unknown => false,
7663            ModuleControlRpcCompletion::Settled => true,
7664            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7665                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7666                info!(
7667                    module_id = %module_id,
7668                    latency_ms,
7669                    "late health.check answer proves the module is alive"
7670                );
7671                match self
7672                    .supervisor
7673                    .record_late_health_answer(&module_id, latency_ms)
7674                {
7675                    Ok(true) => {}
7676                    Ok(false) => debug!(
7677                        module_id = %module_id,
7678                        latency_ms,
7679                        "late health.check answer has no active supervisor snapshot"
7680                    ),
7681                    Err(err) => warn!(
7682                        module_id = %module_id,
7683                        latency_ms,
7684                        error = %err,
7685                        "failed to record late health.check answer"
7686                    ),
7687                }
7688                true
7689            }
7690        }
7691    }
7692
7693    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7694    /// the module connection whose frame is being handled, or to the client that
7695    /// relay was opened for.
7696    ///
7697    /// This runs on the MODULE connection's frame handler, where returning `Err`
7698    /// ends that connection -- and a module connection carries every client's
7699    /// routes to that module, so ending it costs the whole fleet its tools.
7700    /// `ConnectionClosing` carries the id of the connection that is closing, and
7701    /// when that id is a CLIENT's, the condition is entirely about that one
7702    /// client's route.open. A client-scoped condition has no authority over a
7703    /// shared module connection, so it is logged and the single relay is dropped:
7704    /// the client is going away, and `complete_pending_relay` already removed the
7705    /// relay before failing, so there is nothing left to settle. Anything that
7706    /// relay still reserved is released by that client's own connection teardown,
7707    /// which is already under way -- that is what "closing" means.
7708    ///
7709    /// Every other failure is a statement about THIS connection and stays fatal:
7710    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7711    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7712    /// frames correctly.
7713    fn refuse_to_end_module_connection_for_a_client(
7714        &self,
7715        module_connection_id: ConnectionId,
7716        corr: u64,
7717        err: ForwardingError,
7718    ) -> Result<(), RouterError> {
7719        if let ForwardingError::ConnectionClosing { connection_id } = err {
7720            if connection_id != module_connection_id {
7721                warn!(
7722                    module_connection_id = module_connection_id.get(),
7723                    client_connection_id = connection_id.get(),
7724                    corr,
7725                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7726                );
7727                return Ok(());
7728            }
7729        }
7730        Err(RouterError::Forwarding(err))
7731    }
7732
7733    fn handle_module_relay_response(
7734        &self,
7735        connection_id: ConnectionId,
7736        frame: Frame,
7737    ) -> Result<Vec<Frame>, RouterError> {
7738        let mut secondary_error = None;
7739        let outcome = match frame.header.ty {
7740            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7741                Ok(probe) if probe.op == "route.bind" => {
7742                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7743                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7744                            RouteBindRelayOutcome::Accepted
7745                        }
7746                        Ok(other) => {
7747                            let message =
7748                                format!("route.bind response carried unexpected body: {other:?}");
7749                            secondary_error = Some(control_error_frame(
7750                                &frame,
7751                                "invalid_control_body",
7752                                message.clone(),
7753                            )?);
7754                            RouteBindRelayOutcome::ModuleGone(message)
7755                        }
7756                        Err(err) => {
7757                            let message = format!("malformed route.bind response body: {err}");
7758                            secondary_error = Some(control_error_frame(
7759                                &frame,
7760                                "invalid_control_body",
7761                                message.clone(),
7762                            )?);
7763                            RouteBindRelayOutcome::ModuleGone(message)
7764                        }
7765                    }
7766                }
7767                Ok(probe) => {
7768                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7769                    {
7770                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7771                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7772                            "malformed {} response body: {err}",
7773                            probe.op
7774                        )),
7775                    };
7776                    let completion = self
7777                        .forwarding
7778                        .complete_module_control_rpc(
7779                            connection_id,
7780                            frame.header.corr,
7781                            Some(&probe.op),
7782                            outcome,
7783                        )
7784                        .map_err(RouterError::Forwarding)?;
7785                    if !self.observe_module_control_completion(completion) {
7786                        debug!(
7787                            connection_id = connection_id.get(),
7788                            corr = frame.header.corr,
7789                            op = %probe.op,
7790                            "dropping late or unknown module-control RPC response"
7791                        );
7792                    }
7793                    return Ok(Vec::new());
7794                }
7795                Err(err) => {
7796                    if let Some(expected_op) = self
7797                        .forwarding
7798                        .pending_module_control_op(connection_id, frame.header.corr)
7799                        .map_err(RouterError::Forwarding)?
7800                    {
7801                        let completion = self
7802                            .forwarding
7803                            .complete_module_control_rpc(
7804                                connection_id,
7805                                frame.header.corr,
7806                                None,
7807                                ModuleControlRpcOutcome::MalformedResponse(format!(
7808                                    "malformed {expected_op} response body: {err}"
7809                                )),
7810                            )
7811                            .map_err(RouterError::Forwarding)?;
7812                        if !self.observe_module_control_completion(completion) {
7813                            debug!(
7814                                connection_id = connection_id.get(),
7815                                corr = frame.header.corr,
7816                                "dropping late malformed module-control RPC response"
7817                            );
7818                        }
7819                        return Ok(Vec::new());
7820                    }
7821                    let message = format!("malformed route.bind response body: {err}");
7822                    secondary_error = Some(control_error_frame(
7823                        &frame,
7824                        "invalid_control_body",
7825                        message.clone(),
7826                    )?);
7827                    RouteBindRelayOutcome::ModuleGone(message)
7828                }
7829            },
7830            FrameType::Error => {
7831                if self
7832                    .forwarding
7833                    .pending_module_control_op(connection_id, frame.header.corr)
7834                    .map_err(RouterError::Forwarding)?
7835                    .is_some()
7836                {
7837                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7838                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7839                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7840                            "malformed module-control ERROR body: {err}"
7841                        )),
7842                    };
7843                    let completion = self
7844                        .forwarding
7845                        .complete_module_control_rpc(
7846                            connection_id,
7847                            frame.header.corr,
7848                            None,
7849                            outcome,
7850                        )
7851                        .map_err(RouterError::Forwarding)?;
7852                    if !self.observe_module_control_completion(completion) {
7853                        debug!(
7854                            connection_id = connection_id.get(),
7855                            corr = frame.header.corr,
7856                            "dropping late or unknown module-control RPC error"
7857                        );
7858                    }
7859                    return Ok(Vec::new());
7860                }
7861                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7862                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7863                    Err(err) => {
7864                        let message = format!("malformed route.bind ERROR body: {err}");
7865                        secondary_error = Some(control_error_frame(
7866                            &frame,
7867                            "invalid_control_body",
7868                            message.clone(),
7869                        )?);
7870                        RouteBindRelayOutcome::ModuleGone(message)
7871                    }
7872                }
7873            }
7874            ty => {
7875                return Ok(vec![control_error_frame(
7876                    &frame,
7877                    "unsupported_control_frame",
7878                    format!("unsupported module channel-0 frame {ty:?}"),
7879                )?])
7880            }
7881        };
7882
7883        let settled =
7884            self.forwarding
7885                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7886        let completion = match settled {
7887            Ok(completion) => completion,
7888            Err(err) => {
7889                self.refuse_to_end_module_connection_for_a_client(
7890                    connection_id,
7891                    frame.header.corr,
7892                    err,
7893                )?;
7894                return Ok(secondary_error.into_iter().collect());
7895            }
7896        };
7897        if let Some(target) = completion.abandoned.as_ref() {
7898            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7899        }
7900        if !completion.settled {
7901            debug!(
7902                connection_id = connection_id.get(),
7903                corr = frame.header.corr,
7904                frame_type = ?frame.header.ty,
7905                "dropping late or unknown route.bind relay response"
7906            );
7907        }
7908        Ok(secondary_error.into_iter().collect())
7909    }
7910
7911    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7912        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7913        // GOODBYE ends the connection's logical session even when its socket
7914        // stays open. Use disconnect teardown so verdicts, client notices and
7915        // scope authority are released at the same lifecycle boundary.
7916        self.cleanup_connection_with_end_reason(
7917            connection_id,
7918            RegistrationEndReason::ExplicitGoodbye,
7919        )
7920        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7921        Ok(Vec::new())
7922    }
7923}
7924
7925impl Default for ControlHandler {
7926    fn default() -> Self {
7927        Self::new(Arc::new(Registry::default()))
7928    }
7929}
7930
7931impl crate::supervise::SwapPromotionObserver for ControlHandler {
7932    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7933        self.apply_registration_capabilities(registration);
7934    }
7935}
7936
7937fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7938    CapabilityRequirementStatus {
7939        consumer: status.consumer,
7940        capability: status.capability,
7941        need: match status.need {
7942            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7943            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7944        },
7945        verdict: status.verdict.as_str().to_string(),
7946        episode_seq: status.episode_seq,
7947        config_satisfiable: status.config_satisfiable,
7948        runtime_available: status.runtime_available,
7949        detail: status.detail,
7950    }
7951}
7952
7953fn append_capability_problem_detail(
7954    detail: Option<String>,
7955    capability_detail: Option<String>,
7956) -> Option<String> {
7957    match (detail, capability_detail) {
7958        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7959        (Some(detail), None) => Some(detail),
7960        (None, Some(capability_detail)) => Some(capability_detail),
7961        (None, None) => None,
7962    }
7963}
7964
7965fn subc_ops() -> Vec<String> {
7966    SUBC_CONTROL_OPS
7967        .iter()
7968        .map(|op| (*op).to_string())
7969        .collect()
7970}
7971
7972fn module_subc_ops() -> Vec<String> {
7973    SUBC_CONTROL_OPS
7974        .iter()
7975        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7976        .map(|op| (*op).to_string())
7977        .collect()
7978}
7979
7980#[cfg(test)]
7981fn module_baseline_control_ops() -> Vec<String> {
7982    MODULE_BASELINE_CONTROL_OPS
7983        .iter()
7984        .map(|op| (*op).to_string())
7985        .collect()
7986}
7987
7988fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7989    let mut seen = HashSet::new();
7990    let mut effective = Vec::new();
7991    for op in MODULE_BASELINE_CONTROL_OPS {
7992        if seen.insert((*op).to_string()) {
7993            effective.push((*op).to_string());
7994        }
7995    }
7996    for op in declared.unwrap_or_default() {
7997        if seen.insert(op.clone()) {
7998            effective.push(op);
7999        }
8000    }
8001    effective
8002}
8003
8004fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8005    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8006}
8007
8008fn target_module_id(target: &RouteTarget) -> &str {
8009    match target {
8010        RouteTarget::ToolProvider { module_id }
8011        | RouteTarget::ManagementSurface { module_id }
8012        | RouteTarget::InternalService { module_id, .. } => module_id,
8013    }
8014}
8015
8016fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8017    roles.iter().any(|role| match (target, role) {
8018        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8019        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8020        (
8021            RouteTarget::InternalService { service_id, .. },
8022            ProviderRole::InternalService {
8023                service_id: provided,
8024                ..
8025            },
8026        ) => service_id == provided,
8027        _ => false,
8028    })
8029}
8030
8031fn is_routable_role(role: &ProviderRole) -> bool {
8032    matches!(
8033        role,
8034        ProviderRole::ToolProvider { .. }
8035            | ProviderRole::ManagementSurface { .. }
8036            | ProviderRole::InternalService { .. }
8037    )
8038}
8039
8040#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8041enum ControlRequestBodyError {
8042    UnknownOp,
8043    InvalidBody,
8044}
8045
8046#[derive(Debug, Deserialize)]
8047struct ControlOpProbe {
8048    op: String,
8049}
8050
8051/// Channel-0 push ops this subc version understands. A push whose `op` is not in
8052/// this set is treated as a forward-compat unknown and ignored rather than errored.
8053const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8054
8055fn is_known_module_push_op(body: &[u8]) -> bool {
8056    serde_json::from_slice::<ControlOpProbe>(body)
8057        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8058        .unwrap_or(false)
8059}
8060
8061fn is_known_module_request_op(body: &[u8]) -> bool {
8062    serde_json::from_slice::<ControlOpProbe>(body)
8063        .map(|probe| is_module_to_subc_op(&probe.op))
8064        .unwrap_or(false)
8065}
8066
8067fn is_module_to_subc_op(op: &str) -> bool {
8068    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8069}
8070
8071fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8072    debug!(
8073        op = %op,
8074        connection_id = connection_id.get(),
8075        corr,
8076        "control dispatch"
8077    );
8078}
8079
8080fn log_slow_control_dispatch(
8081    dispatch_started_at: Option<StdInstant>,
8082    op: &'static str,
8083    connection_id: ConnectionId,
8084    corr: u64,
8085    route_open_timings: Option<&RouteOpenTimings>,
8086) {
8087    let Some(dispatch_started_at) = dispatch_started_at else {
8088        return;
8089    };
8090    // Use the same monotonic clock as the phase timers and relay deadline.
8091    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8092    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8093        if let Some(timings) = route_open_timings {
8094            warn!(
8095                op = %op,
8096                connection_id = connection_id.get(),
8097                corr,
8098                elapsed_ms = elapsed.as_millis() as u64,
8099                module_id = ?timings.module_id,
8100                principal = %timings.principal,
8101                admission_ms = timings.admission.as_millis() as u64,
8102                project_root_ms = timings.project_root.as_millis() as u64,
8103                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8104                bind_send_ms = timings.bind_send.as_millis() as u64,
8105                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8106                "slow control dispatch"
8107            );
8108            return;
8109        }
8110        warn!(
8111            op = %op,
8112            connection_id = connection_id.get(),
8113            corr,
8114            elapsed_ms = elapsed.as_millis() as u64,
8115            "slow control dispatch"
8116        );
8117    }
8118}
8119
8120/// A test stand-in for the OS caller lookup, given the recorded endpoints.
8121#[cfg(test)]
8122type TestCallerLookup =
8123    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8124
8125/// Every new client operation needs an explicit privilege decision here.
8126fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8127    match request {
8128        ClientControlRequest::SupervisorRestart { module_id, .. } => {
8129            Some(PrivilegedOperation::Restart { module_id })
8130        }
8131        ClientControlRequest::SupervisorSwap { module_id, .. } => {
8132            Some(PrivilegedOperation::Swap { module_id })
8133        }
8134        ClientControlRequest::SupervisorReload { module_id } => {
8135            Some(PrivilegedOperation::Reload { module_id })
8136        }
8137        ClientControlRequest::SupervisorRescan { preview } => {
8138            if *preview {
8139                None
8140            } else {
8141                Some(PrivilegedOperation::Rescan)
8142            }
8143        }
8144        // Only the well-formed apply asks for approval. A preview changes
8145        // nothing, and any other combination is refused before it touches
8146        // anything, so prompting the operator for it would ask about a request
8147        // that cannot run.
8148        ClientControlRequest::SupervisorRescanChecked {
8149            preview,
8150            candidate,
8151            expect,
8152        } => (!*preview && candidate.is_none() && expect.is_some())
8153            .then_some(PrivilegedOperation::Rescan),
8154        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8155            Some(PrivilegedOperation::SetEnabled {
8156                module_id,
8157                enabled: *enabled,
8158            })
8159        }
8160        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8161            Some(PrivilegedOperation::ReleaseReserved { module_id })
8162        }
8163        ClientControlRequest::ServerDescribe {}
8164        // The announcement uses the same client admission as supervisor.restart,
8165        // but cannot stop anything itself and needs no operator confirmation.
8166        | ClientControlRequest::DaemonShutdownIntent { .. }
8167        | ClientControlRequest::CatalogList { .. }
8168        | ClientControlRequest::RouteOpen { .. }
8169        | ClientControlRequest::RoutePoll { .. }
8170        | ClientControlRequest::SupervisorList {}
8171        | ClientControlRequest::SupervisorHealth {}
8172        | ClientControlRequest::SupervisorHealthProbe { .. }
8173        | ClientControlRequest::SupervisorRoutes { .. }
8174        | ClientControlRequest::SupervisorProvenance { .. }
8175        | ClientControlRequest::SupervisorStderrTail { .. }
8176        | ClientControlRequest::SupervisorTerminals { .. }
8177        | ClientControlRequest::SupervisorSpawnSnapshot {}
8178        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8179    }
8180}
8181
8182fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8183    match request {
8184        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8185        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8186        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8187        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8188        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8189        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8190        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8191        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8192        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8193        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8194        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8195        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8196        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8197        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8198        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8199        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8200        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8201        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8202        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8203        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8204        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8205    }
8206}
8207
8208/// The caller of one `route.open`, as its `route.open refused` line names it.
8209///
8210/// A connection id alone cannot name a caller after the fact: ids restart at
8211/// every daemon restart, so the same number belongs to different modules
8212/// before and after one. These three fields are what the refused line needs
8213/// to be attributable on its own, and they are rendered with the same helpers
8214/// as the `route.open accepted` line so both directions grep alike.
8215struct RouteOpenCaller {
8216    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
8217    /// when the request presented a consumer identity whose launch nonce did
8218    /// not verify. The claimed module id is never printed as a principal.
8219    principal: String,
8220    /// `none` for an unscoped open; the admitted stamp's label once scope
8221    /// admission issued one; before that, the requested selector's label.
8222    scope: String,
8223    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
8224    /// because it is a string the caller chose.
8225    harness: String,
8226}
8227
8228impl RouteOpenCaller {
8229    fn new(
8230        principal: Option<&Principal>,
8231        scope: Option<&ScopeSelector>,
8232        identity: &BindIdentity,
8233    ) -> Self {
8234        Self {
8235            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8236            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8237            harness: bounded_log_text(&identity.harness),
8238        }
8239    }
8240
8241    /// For a refusal that has no parsed request at hand: names whatever the
8242    /// frame still says, and `unknown` where it says nothing parseable.
8243    fn unknown() -> Self {
8244        Self {
8245            principal: "unknown".to_string(),
8246            scope: "unknown".to_string(),
8247            harness: "unknown".to_string(),
8248        }
8249    }
8250}
8251
8252/// Longest client-chosen text (a scope ref, a harness) a route.open log line
8253/// prints before cutting it.
8254const MAX_LOGGED_TEXT_CHARS: usize = 64;
8255
8256/// `value` Debug-quoted, so control characters land escaped and cannot forge
8257/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
8258/// `…` inside the quotes so one request cannot flood the line.
8259fn bounded_log_text(value: &str) -> String {
8260    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8261    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8262        bounded.push('…');
8263    }
8264    format!("{bounded:?}")
8265}
8266
8267/// The `scope=` field of the `route.open accepted` line, from the stamp that
8268/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
8269/// so a ref holding control characters cannot forge a log line, and cut to 64
8270/// characters with a trailing `…` to bound the line. The bind proof and the
8271/// scope's attributes never appear.
8272fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8273    use subc_protocol::scope::ScopeKind;
8274    let kind = match stamp.kind {
8275        ScopeKind::Head => "head",
8276        ScopeKind::Worker => "worker",
8277        ScopeKind::Ephemeral => "ephemeral",
8278        ScopeKind::Federated => "federated",
8279        _ => "unknown",
8280    };
8281    format!(
8282        "{kind}/{}/{}@{}",
8283        principal_label(&stamp.owner),
8284        bounded_log_text(&stamp.scope_ref),
8285        stamp.scope_epoch
8286    )
8287}
8288
8289/// The `scope=` field of a `route.open refused` line for an open refused
8290/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
8291/// The kind is not known yet, so the literal `requested` stands in its place.
8292/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
8293/// owner it has not been checked against the configured modules: a quoted
8294/// owner means the value was never validated. The bind proof never appears.
8295fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8296    let epoch = selector
8297        .scope_epoch
8298        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8299    format!(
8300        "requested/{}/{}@{epoch}",
8301        bounded_log_text(&principal_label(&selector.owner)),
8302        bounded_log_text(&selector.scope_ref),
8303    )
8304}
8305
8306fn principal_label(principal: &Principal) -> String {
8307    match principal {
8308        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8309        Principal::Direct => "direct".to_string(),
8310        other => format!("{other:?}"),
8311    }
8312}
8313
8314fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8315    match request {
8316        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8317        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8318        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8319        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8320        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8321        ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8322        _ => "unsupported",
8323    }
8324}
8325
8326fn parse_client_control_request(
8327    body: &[u8],
8328) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8329    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8330        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8331            Ok(probe)
8332                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8333                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8334            {
8335                ControlRequestBodyError::InvalidBody
8336            }
8337            Ok(_) => ControlRequestBodyError::UnknownOp,
8338            Err(_) => ControlRequestBodyError::InvalidBody,
8339        };
8340        (err, classification)
8341    })
8342}
8343
8344fn parse_module_control_request_from_module(
8345    body: &[u8],
8346) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8347    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8348        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8349            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8350            Ok(_) => ControlRequestBodyError::UnknownOp,
8351            Err(_) => ControlRequestBodyError::InvalidBody,
8352        };
8353        (err, classification)
8354    })
8355}
8356
8357#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8358enum ProviderRoleKind {
8359    ToolProvider,
8360    PipelineStage,
8361    ManagementSurface,
8362    InternalService,
8363}
8364
8365fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8366    match role {
8367        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8368        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8369        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8370        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8371    }
8372}
8373
8374fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8375    roles.iter().map(provider_role_kind).collect()
8376}
8377
8378/// Most refused scope records named individually in the log per sync; the
8379/// `refused` count on the accepted line is always complete.
8380const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8381
8382/// Per-outcome counts of one accepted `scope.sync`, for its log line.
8383#[derive(Debug, Default, PartialEq, Eq)]
8384struct ScopeOutcomeCounts {
8385    created: usize,
8386    replaced: usize,
8387    updated: usize,
8388    unchanged: usize,
8389    refused: usize,
8390}
8391
8392impl ScopeOutcomeCounts {
8393    fn of(results: &[ScopeRecordResult]) -> Self {
8394        let mut counts = Self::default();
8395        for result in results {
8396            let slot = match result.outcome {
8397                ScopeRecordOutcome::Created => &mut counts.created,
8398                ScopeRecordOutcome::Replaced => &mut counts.replaced,
8399                ScopeRecordOutcome::Updated => &mut counts.updated,
8400                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8401                ScopeRecordOutcome::Refused => &mut counts.refused,
8402            };
8403            *slot += 1;
8404        }
8405        counts
8406    }
8407}
8408
8409#[cfg(test)]
8410mod scope_outcome_count_tests {
8411    use super::*;
8412
8413    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8414        ScopeRecordResult {
8415            scope_ref: "r".to_string(),
8416            scope_epoch: 1,
8417            outcome,
8418            code: None,
8419            message: None,
8420            version: None,
8421            parent_state: None,
8422        }
8423    }
8424
8425    /// Each outcome lands in its own count, so a refused record can never be
8426    /// hidden inside the total the log already printed.
8427    #[test]
8428    fn every_outcome_is_counted_in_its_own_field() {
8429        let results = [
8430            result(ScopeRecordOutcome::Created),
8431            result(ScopeRecordOutcome::Created),
8432            result(ScopeRecordOutcome::Replaced),
8433            result(ScopeRecordOutcome::Updated),
8434            result(ScopeRecordOutcome::Unchanged),
8435            result(ScopeRecordOutcome::Refused),
8436            result(ScopeRecordOutcome::Refused),
8437            result(ScopeRecordOutcome::Refused),
8438        ];
8439        assert_eq!(
8440            ScopeOutcomeCounts::of(&results),
8441            ScopeOutcomeCounts {
8442                created: 2,
8443                replaced: 1,
8444                updated: 1,
8445                unchanged: 1,
8446                refused: 3,
8447            }
8448        );
8449    }
8450}
8451
8452/// Return whether a catalog change can create a newly violating live route.
8453/// Removing an attested claim is intentionally excluded: it makes fewer routes
8454/// forbidden and therefore must leave the existing route census untouched.
8455fn capability_census_trigger(
8456    old: Option<&CapabilityDeclarations>,
8457    new: Option<&CapabilityDeclarations>,
8458) -> bool {
8459    let old_provides = old
8460        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8461        .unwrap_or_default();
8462    let old_denies = old
8463        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8464        .unwrap_or_default();
8465    let new = new.cloned().unwrap_or(CapabilityDeclarations {
8466        provides: Vec::new(),
8467        requires: Vec::new(),
8468        must_never_reach: Vec::new(),
8469    });
8470
8471    new.provides
8472        .iter()
8473        .any(|capability| !old_provides.contains(capability))
8474        || new
8475            .must_never_reach
8476            .iter()
8477            .any(|capability| !old_denies.contains(capability))
8478}
8479
8480/// Find the first capability an attested opener denies that an attested target
8481/// claims. Both manifests are live registry records, never cached or client data.
8482fn denied_capability<'a>(
8483    opening_manifest: &'a ModuleManifest,
8484    target_manifest: &ModuleManifest,
8485) -> Option<&'a str> {
8486    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8487    let target_capabilities = target_manifest.capabilities.as_ref()?;
8488    opening_capabilities
8489        .must_never_reach
8490        .iter()
8491        .find(|denied| {
8492            target_capabilities
8493                .provides
8494                .iter()
8495                .any(|provided| provided == *denied)
8496        })
8497        .map(String::as_str)
8498}
8499
8500fn catalog_update_frozen_field_message(
8501    registered: &ModuleManifest,
8502    provides: &[ProviderRole],
8503) -> Option<String> {
8504    let old_has_provides = !registered.provides.is_empty();
8505    let new_has_provides = !provides.is_empty();
8506    if old_has_provides != new_has_provides {
8507        return Some(format!(
8508            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8509            registered.module_id
8510        ));
8511    }
8512
8513    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
8514        return Some(format!(
8515            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8516            registered.module_id
8517        ));
8518    }
8519
8520    let registered_concurrency = manifest_concurrency(registered);
8521    let mut candidate = registered.clone();
8522    candidate.provides = provides.to_vec();
8523    let candidate_concurrency = manifest_concurrency(&candidate);
8524    if candidate_concurrency != registered_concurrency {
8525        return Some(format!(
8526            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8527            registered.module_id, registered_concurrency, candidate_concurrency
8528        ));
8529    }
8530
8531    // control_ops live beside the manifest in the HELLO body, not inside
8532    // ModuleManifest, so a provides-only catalog.update cannot change them.
8533    None
8534}
8535
8536fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8537    manifest.provides.iter().any(is_routable_role)
8538}
8539
8540/// Returns the routable-provider concurrency subc should enforce for this manifest.
8541///
8542/// ToolProvider and ManagementSurface store their delivery concurrency directly.
8543/// InternalService has no role-specific concurrency field, so it retains the
8544/// existing ModuleManaged default for backward compatibility.
8545fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8546    manifest
8547        .provides
8548        .iter()
8549        .find_map(|provider| match provider {
8550            ProviderRole::ToolProvider { concurrency, .. }
8551            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8552            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8553        })
8554        .unwrap_or(Concurrency::ModuleManaged)
8555}
8556
8557/// True when the manifest carries a ManagementSurface role whose concurrency
8558/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
8559/// bytes because the typed manifest deliberately erases that distinction: the
8560/// default exists for wire compatibility, and this probe exists so the default
8561/// stays observable. Any parse irregularity returns false -- the caller only
8562/// logs, and a malformed body already failed registration upstream.
8563fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8564    let has_management_surface = manifest
8565        .provides
8566        .iter()
8567        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8568    if !has_management_surface {
8569        return false;
8570    }
8571    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8572        return false;
8573    };
8574    let Some(provides) = raw
8575        .get("manifest")
8576        .and_then(|manifest| manifest.get("provides"))
8577        .and_then(serde_json::Value::as_array)
8578    else {
8579        return false;
8580    };
8581    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
8582    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
8583    // against the management_surface_manifest_without_concurrency golden, not
8584    // recalled (the externally-tagged guess was this function's first bug).
8585    provides.iter().any(|role| {
8586        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8587            && role.get("concurrency").is_none()
8588    })
8589}
8590
8591fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8592    if peer_version != PROTOCOL_VERSION {
8593        return Err(format!(
8594            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8595        ));
8596    }
8597    Ok(PROTOCOL_VERSION)
8598}
8599
8600fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8601    Frame::build_with_version(
8602        response_version(frame),
8603        FrameType::Pong,
8604        frame.header.flags,
8605        0,
8606        0,
8607        frame.header.corr,
8608        Vec::new(),
8609    )
8610    .map_err(RouterError::FrameBuild)
8611}
8612
8613pub(crate) fn control_error_frame(
8614    frame: &Frame,
8615    code: &'static str,
8616    message: impl Into<String>,
8617) -> Result<Frame, RouterError> {
8618    control_error_body_frame(
8619        frame,
8620        ErrorBody {
8621            code: code.to_string(),
8622            message: message.into(),
8623            detail: None,
8624        },
8625    )
8626}
8627
8628/// A channel-0 ERROR for a refusal the daemon decided itself: every
8629/// route.open refusal, the drain `module_reloading` and scope-change/end
8630/// bodies, spawn-subscription and supervisor refusals. Carries the
8631/// daemon-origin bit, which is what lets a client treat `module_reloading`
8632/// as proof that its request never reached a module. Response, push,
8633/// goodbye and relay-request frames keep the plain control flags in their
8634/// own builders.
8635fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8636    control_error_frame_with_flags(frame, error, daemon_error_flags())
8637}
8638
8639/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8640/// refused route.bind answered as a route.open refusal, or a refused
8641/// module-control RPC. Plain control flags — the code and message are
8642/// module-chosen, so the frame must not claim daemon authorship, even when
8643/// the module picks a daemon code like `module_reloading`.
8644fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8645    control_error_frame_with_flags(frame, error, control_flags())
8646}
8647
8648fn control_error_frame_with_flags(
8649    frame: &Frame,
8650    error: ErrorBody,
8651    flags: Flags,
8652) -> Result<Frame, RouterError> {
8653    let body = serde_json::to_vec(&error).map_err(|err| {
8654        RouterError::backend(
8655            0,
8656            frame.header.corr,
8657            format!("failed to encode control ERROR: {err}"),
8658        )
8659    })?;
8660
8661    Frame::build_with_version(
8662        response_version(frame),
8663        FrameType::Error,
8664        flags,
8665        0,
8666        0,
8667        frame.header.corr,
8668        body,
8669    )
8670    .map_err(RouterError::FrameBuild)
8671}
8672
8673pub(crate) fn control_response_body_frame<T: Serialize>(
8674    frame: &Frame,
8675    reply: &T,
8676    label: &'static str,
8677) -> Result<Frame, RouterError> {
8678    let body = serde_json::to_vec(reply).map_err(|err| {
8679        RouterError::backend(
8680            0,
8681            frame.header.corr,
8682            format!("failed to encode {label}: {err}"),
8683        )
8684    })?;
8685
8686    Frame::build_with_version(
8687        response_version(frame),
8688        FrameType::Response,
8689        control_flags(),
8690        0,
8691        0,
8692        frame.header.corr,
8693        body,
8694    )
8695    .map_err(RouterError::FrameBuild)
8696}
8697
8698/// Map a forwarding failure to the wire code a client sees.
8699///
8700/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8701/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8702/// chosen here decides whether a caller retries or gives up.
8703///
8704/// That makes attribution the load-bearing property, not merely having a code. A
8705/// permanent fault published as a retryable one produces a fleet-wide retry storm
8706/// against something that can never recover; a transient fault published as
8707/// permanent gives up on work that would have succeeded. Both look correct in a
8708/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8709/// the mapping per variant rather than merely asserting that some code exists.
8710///
8711/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8712/// two codes on the same side of the boundary passes it. Measured rather than
8713/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8714/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8715/// a test named for something else that happens to assert the string.
8716///
8717/// That accidental coverage is deliberately left alone rather than promoted to a
8718/// named test, because it guards a property this function does not promise.
8719/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8720/// specific code within a class, so identity is free to change and only the
8721/// partition is a contract. Splitting it out would assert a guarantee nothing
8722/// depends on — and a suite that promises more than the code does is the harder
8723/// thing to correct later, because the next reader cannot tell which assertions
8724/// are load-bearing.
8725///
8726/// Pin identity here the moment a consumer branches on a specific code.
8727fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8728    match err {
8729        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8730        ForwardingError::NoModuleConnection => "target_unavailable",
8731        ForwardingError::ModuleReloading { .. } => "module_reloading",
8732        ForwardingError::ClientRouteChannelExhausted { .. }
8733        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8734        ForwardingError::StaleModuleEndpoint
8735        | ForwardingError::UnknownReservation { .. }
8736        | ForwardingError::ConnectionClosing { .. }
8737        | ForwardingError::ClientEgressClosed { .. }
8738        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8739        // Only a swap candidate's registration can produce this, and it means
8740        // exactly what a second active HELLO for a live id means.
8741        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8742        ForwardingError::RelayCorrelationExhausted
8743        | ForwardingError::RouteOpenBuild(_)
8744        | ForwardingError::Poisoned => "forwarding_error",
8745    }
8746}
8747
8748fn response_version(frame: &Frame) -> u8 {
8749    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8750        frame.header.ver
8751    } else {
8752        PROTOCOL_VERSION
8753    }
8754}
8755
8756fn control_flags() -> Flags {
8757    Flags::new(false, Priority::Passive, false)
8758}
8759
8760/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8761/// channel. The target is the module (a client never saw the route), so this
8762/// takes the module path: delivered late rather than dropped when the module's
8763/// queue is momentarily full, and never closing its connection.
8764fn send_goodbye_target_best_effort(
8765    counters: &DaemonCounters,
8766    target: &GoodbyeTarget,
8767    context: &'static str,
8768) {
8769    let Ok(frame) = Frame::build_with_version(
8770        target.negotiated_ver,
8771        FrameType::Goodbye,
8772        control_flags(),
8773        target.channel,
8774        target.epoch,
8775        0,
8776        Vec::new(),
8777    ) else {
8778        return;
8779    };
8780    crate::forwarding::send_module_route_goodbye(
8781        counters,
8782        &target.sink,
8783        frame,
8784        target.module_id.as_deref(),
8785        context,
8786    );
8787}
8788
8789pub(crate) fn send_route_control_pushes(
8790    forwarding: &ForwardingTable,
8791    routes: Vec<EndpointRoute>,
8792    push: ClientControlPush,
8793) {
8794    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8795    for route in routes {
8796        let target = route.goodbye_target;
8797        if let Some((existing, channels)) = targets
8798            .iter_mut()
8799            .find(|(existing, _)| existing.connection_id == target.connection_id)
8800        {
8801            debug_assert_eq!(
8802                existing.negotiated_ver, target.negotiated_ver,
8803                "one connection cannot negotiate multiple frame versions"
8804            );
8805            if !channels.contains(&target.channel) {
8806                channels.push(target.channel);
8807            }
8808            continue;
8809        }
8810        let channel = target.channel;
8811        targets.push((target, vec![channel]));
8812    }
8813    for (target, mut channels) in targets {
8814        channels.sort_unstable();
8815        let mut push = push.clone();
8816        match &mut push {
8817            ClientControlPush::RouteClosing {
8818                channels: covered, ..
8819            }
8820            | ClientControlPush::RouteClosed {
8821                channels: covered, ..
8822            } => *covered = channels,
8823        }
8824        let body = match serde_json::to_vec(&push) {
8825            Ok(body) => body,
8826            Err(err) => {
8827                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8828                continue;
8829            }
8830        };
8831        let frame = match Frame::build_with_version(
8832            target.negotiated_ver,
8833            FrameType::Push,
8834            control_flags(),
8835            0,
8836            0,
8837            0,
8838            body.clone(),
8839        ) {
8840            Ok(frame) => frame,
8841            Err(err) => {
8842                warn!(
8843                    route_channel = target.channel,
8844                    error = %err,
8845                    "failed to build route lifecycle control PUSH frame"
8846                );
8847                continue;
8848            }
8849        };
8850        if let Err(err) = target.sink.try_send(frame) {
8851            if target.close_on_delivery_failure() {
8852                warn!(
8853                    target_connection_id = target.connection_id.get(),
8854                    route_channel = target.channel,
8855                    error = %err,
8856                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8857                );
8858                let _ = forwarding.escalate_client_delivery_failure(
8859                    target.connection_id,
8860                    target.channel,
8861                    target.epoch,
8862                    CloseReason::new(
8863                        "route_lifecycle_push_delivery_failed",
8864                        format!(
8865                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8866                            target.channel
8867                        ),
8868                    ),
8869                    crate::forwarding::UndeliveredFrame {
8870                        module_id: target.module_id.as_deref(),
8871                        sink: &target.sink,
8872                    },
8873                );
8874            }
8875        }
8876    }
8877}
8878
8879#[cfg(test)]
8880mod tests {
8881    include!("control/agent_commands_tests.rs");
8882    include!("control/resource_revision_tests.rs");
8883    #[tokio::test]
8884    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8885        let registry = Arc::new(Registry::default());
8886        let handler = ControlHandler::new(registry);
8887        assert_eq!(
8888            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8889            None
8890        );
8891        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8892        hello_via_sink(
8893            &handler,
8894            &ctx,
8895            &mut rx,
8896            hello_frame("daemon", PROTOCOL_VERSION, 1),
8897        )
8898        .await;
8899        assert_eq!(
8900            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8901            Some("daemon".into())
8902        );
8903        assert_eq!(
8904            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8905            None
8906        );
8907        assert_eq!(
8908            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8909            None,
8910            "a registered module is not necessarily supervised"
8911        );
8912        assert_eq!(
8913            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8914            None,
8915            "aggregate provenance has no single target"
8916        );
8917        assert_eq!(
8918            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8919            None
8920        );
8921    }
8922
8923    mod peer_manifest {
8924        include!("peer_manifest_tests.rs");
8925    }
8926    mod activity_read {
8927        include!("activity_read_tests.rs");
8928    }
8929
8930    mod lifecycle_activity {
8931        include!("control_lifecycle_tests.rs");
8932    }
8933    #[cfg(unix)]
8934    #[tokio::test]
8935    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8936        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8937        let path = dir.join("subc.jsonc");
8938        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8939            "program":"/bin/sleep","args":["60"],"protocol":"none",
8940            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8941        }}}).to_string()).unwrap();
8942        let mut configured = crate::daemon_config::load(&path)
8943            .unwrap()
8944            .unwrap()
8945            .modules
8946            .pop()
8947            .unwrap();
8948        let registry = std::sync::Arc::new(crate::Registry::default());
8949        let handle = crate::SupervisorHandle::new();
8950        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8951            .with_handle(handle.clone());
8952        let module = supervisor
8953            .supervise_configured_with_health(
8954                configured.module_spec(),
8955                true,
8956                configured.health.clone(),
8957                None,
8958                configured.restart,
8959            )
8960            .unwrap();
8961        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8962        let before = module.status().unwrap().pid;
8963        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8964        configured.health.cadence = std::time::Duration::from_secs(3600);
8965        let health_only = handler
8966            .reconcile_supervised_modules(
8967                &supervisor,
8968                vec![configured.clone()],
8969                false,
8970                Principal::Direct,
8971            )
8972            .await
8973            .unwrap();
8974        assert!(
8975            health_only.0.changed_pending_reload.is_empty(),
8976            "health policy is already applied live"
8977        );
8978        assert_eq!(module.status().unwrap().pid, before);
8979        assert_eq!(
8980            module.configuration().unwrap().1.http,
8981            configured.health.http
8982        );
8983        configured.args = vec!["61".into()];
8984        let launch = handler
8985            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8986            .await
8987            .unwrap();
8988        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8989        assert_eq!(
8990            module.status().unwrap().pid,
8991            before,
8992            "a launch edit is stored until reload"
8993        );
8994        module.drain().await.unwrap();
8995    }
8996    use cortexkit_test_support::ScratchDir;
8997    use std::{
8998        collections::BTreeMap,
8999        fmt,
9000        path::PathBuf,
9001        sync::{Arc, Mutex},
9002        time::Duration,
9003    };
9004
9005    use serde_json::{json, Value};
9006    use subc_protocol::{
9007        manifest::{
9008            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9009            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9010        },
9011        session::HealthStatus,
9012        FrameType,
9013    };
9014
9015    use super::*;
9016    use crate::{
9017        forwarding::{DataRoute, DataRouteState},
9018        registry::ChannelState,
9019        router::FrameSink,
9020        stderr_tail::DEFAULT_MAX_LINE_BYTES,
9021        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9022        RouteCtx, Router,
9023    };
9024    use tokio::{
9025        sync::mpsc,
9026        time::{sleep, Instant},
9027    };
9028    use tracing::{
9029        field::{Field, Visit},
9030        Event, Subscriber,
9031    };
9032    use tracing_subscriber::{layer::Context, prelude::*, Layer};
9033
9034    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
9035    ///
9036    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
9037    /// is only populated for `tests/*.rs` integration test binaries -- this file
9038    /// compiles as part of the library target, which gets neither. This test's
9039    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
9040    /// and the sibling binary lives two directories up at
9041    /// `<target-dir>/<profile>/fake-aft-stub`.
9042    ///
9043    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
9044    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
9045    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
9046    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
9047    /// `NotFound`, which reads as a broken test rather than an unbuilt
9048    /// dependency -- so state the cause and the remedy instead. Deliberately a
9049    /// panic and not a silent skip: a test that quietly passes when it could not
9050    /// run is worse than one that fails, because it reports health it never
9051    /// verified.
9052    fn fake_aft_stub_path() -> PathBuf {
9053        let mut path = std::env::current_exe().expect("current_exe available in tests");
9054        path.pop(); // .../deps/
9055        path.pop(); // .../<profile>/
9056        path.push(if cfg!(windows) {
9057            "fake-aft-stub.exe"
9058        } else {
9059            "fake-aft-stub"
9060        });
9061        assert!(
9062            path.exists(),
9063            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9064             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9065            path.display()
9066        );
9067        path
9068    }
9069
9070    /// Whether clients retry `code` in place: the predicate itself, never a copy
9071    /// of its set. A copied list breaks silently when a code is added to or
9072    /// removed from the real one, and a stale copy here would let exactly the
9073    /// failure this test exists to catch pass.
9074    fn client_retries(code: &str) -> bool {
9075        subc_protocol::error_codes::is_retryable_route_open(code)
9076    }
9077
9078    /// A code is not a label — clients branch on it, so publishing the wrong KIND
9079    /// of failure is worse than publishing none. A permanent fault dressed as
9080    /// retryable makes every client in the fleet retry forever against something
9081    /// that cannot recover; a transient fault dressed as permanent abandons work
9082    /// that would have succeeded.
9083    ///
9084    /// Asserting "a code exists" cannot catch either, because the string is free
9085    /// to say anything. This enumerates every variant and pins which side of the
9086    /// retry boundary it lands on, so a new variant must be classified here
9087    /// deliberately rather than inheriting whichever arm it was appended to.
9088    #[test]
9089    fn retryability_of_forwarding_codes_matches_the_failure() {
9090        // Transient by nature: the target is booting, reloading, or its endpoint
9091        // was swapped mid-flight. Retrying is how these resolve.
9092        let transient = [
9093            ForwardingError::NoModuleConnection,
9094            ForwardingError::ModuleReloading {
9095                module_id: "m".into(),
9096            },
9097            ForwardingError::StaleModuleEndpoint,
9098            ForwardingError::UnknownReservation {
9099                client_channel: 1,
9100                module_channel: 1,
9101            },
9102            ForwardingError::ConnectionClosing {
9103                connection_id: ConnectionId::new(1),
9104            },
9105            ForwardingError::ClientEgressClosed {
9106                connection_id: ConnectionId::new(1),
9107            },
9108            ForwardingError::ModuleEgressUnavailable {
9109                connection_id: ConnectionId::new(1),
9110            },
9111        ];
9112        for err in transient {
9113            let code = forwarding_error_code(&err);
9114            assert!(
9115                client_retries(code),
9116                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9117            );
9118        }
9119
9120        // Not fixed by retrying. Channel and correlation exhaustion need the
9121        // caller to close routes, and a poisoned lock is a daemon that cannot
9122        // recover at all — the worst thing to advertise as retryable, since every
9123        // client would storm a daemon that will never answer.
9124        let permanent = [
9125            ForwardingError::ConnectionRoleConflict {
9126                connection_id: ConnectionId::new(1),
9127            },
9128            ForwardingError::ClientRouteChannelExhausted {
9129                connection_id: ConnectionId::new(1),
9130            },
9131            ForwardingError::ModuleRouteChannelExhausted {
9132                endpoint: ModuleEndpointId {
9133                    connection_id: ConnectionId::new(1),
9134                    generation: 1,
9135                },
9136            },
9137            ForwardingError::RelayCorrelationExhausted,
9138            ForwardingError::RouteOpenBuild("x".into()),
9139            ForwardingError::Poisoned,
9140        ];
9141        for err in permanent {
9142            let code = forwarding_error_code(&err);
9143            assert!(
9144                !client_retries(code),
9145                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9146            );
9147        }
9148    }
9149
9150    /// The principal is the daemon's answer to "who is calling", and modules
9151    /// branch on it: aft gates bash on it, cerebellum gates browser control,
9152    /// plexus gates connector invocation. So a stamp is an authorization input in
9153    /// another process, not a label — and both possible answers SUCCEED, which is
9154    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
9155    /// first-party capability to something that never proved it; a supervised one
9156    /// stamped `Direct` silently strips a module of capability it is entitled to.
9157    ///
9158    /// Neither shows up in a test that only checks the bind succeeded. Before this
9159    /// test the only coverage was accidental —
9160    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
9161    /// stamped principal on its way past, so narrowing that wire-shape test to its
9162    /// stated subject would have deleted the last assertion on this value. It
9163    /// still asserts the stamp, which is now redundancy rather than the only
9164    /// guard: both fail under the same mutation, and this one names the reason.
9165    /// SCOPE: this handler's supervisor has spawned nothing, so
9166    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
9167    /// is unreachable here. Both assertions below are refusals, and a mutant that
9168    /// refuses everything would satisfy them.
9169    ///
9170    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
9171    /// spawns a supervised consumer, reads its live nonce, and asserts the module
9172    /// observed `principal.kind == "reserved"` carrying that module_id — verified
9173    /// at source rather than assumed, since a citation is a claim about another
9174    /// file and ages like one. Recorded because a harness that structurally
9175    /// cannot reach an arm reports "none" for that arm identically to one that
9176    /// covers it and found nothing.
9177    #[tokio::test]
9178    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9179        let handler = ControlHandler::default();
9180        let frame =
9181            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9182
9183        // Absent consumer_identity is the ordinary case: a human at a terminal, or
9184        // any process holding the connection file. Nothing was proved, so nothing
9185        // may be granted beyond the unattested floor.
9186        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9187        assert_eq!(
9188            stamped,
9189            Principal::Direct,
9190            "a caller that proved nothing must not be stamped as a supervised module"
9191        );
9192
9193        // A claimed module_id with a nonce no supervised child was given is a
9194        // forgery attempt, not a weaker caller: it must be REFUSED rather than
9195        // quietly demoted to Direct, or an impersonation attempt looks identical
9196        // to an ordinary unattested connection.
9197        let forged = handler
9198            .route_open_principal(
9199                &frame,
9200                Some(ConsumerIdentity {
9201                    module_id: "aft".to_string(),
9202                    launch_nonce: "not-a-real-nonce".to_string(),
9203                }),
9204            )
9205            .unwrap();
9206        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9207        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9208    }
9209
9210    /// The test above hands `route_open_principal` an identity it built itself,
9211    /// which proves the stamping rule and nothing about where the identity comes
9212    /// from. The real producer is a wire body, and the two are joined by a serde
9213    /// field name that nothing else asserts.
9214    ///
9215    /// That join fails quietly in one specific way: an unrecognised key is simply
9216    /// absent after parsing, so a renamed or misspelled `consumer_identity`
9217    /// yields `None` and every supervised module silently drops to `Direct`.
9218    /// Capability-wise that is the safe direction, but it surfaces far from its
9219    /// cause — as a module mysteriously refused bash — and it would pass every
9220    /// test that builds its own input.
9221    ///
9222    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
9223    /// would break every client the moment the daemon gains a field, trading a
9224    /// quiet demotion for a hard refusal on additive change. Asserting the join
9225    /// instead means a rename breaks a test here rather than the fleet.
9226    #[test]
9227    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9228        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9229        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9230        let ClientControlRequest::RouteOpen {
9231            consumer_identity, ..
9232        } = parsed
9233        else {
9234            panic!("route.open body must parse as RouteOpen");
9235        };
9236        assert_eq!(
9237            consumer_identity,
9238            Some(ConsumerIdentity {
9239                module_id: "aft".to_string(),
9240                launch_nonce: "n".to_string(),
9241            }),
9242            "the wire field name must reach the value route_open_principal reads"
9243        );
9244    }
9245
9246    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9247        ModuleManifest::builder(module_id, "0.1.0")
9248            .protocol_ver(protocol_ver)
9249            .provides(vec![ProviderRole::ToolProvider {
9250                tools: vec![Tool {
9251                    name: "read".to_string(),
9252                    description: None,
9253                    execution_mode: ExecutionMode::Pure,
9254                    schema: json!({"type": "object"}),
9255                }],
9256                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9257                concurrency: Concurrency::ModuleManaged,
9258                emits_push: true,
9259                sub_supervises: true,
9260            }])
9261            .build()
9262    }
9263
9264    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9265        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9266    }
9267
9268    fn hello_frame_with_control_ops(
9269        module_id: &str,
9270        protocol_ver: u8,
9271        corr: u64,
9272        control_ops: Option<Vec<String>>,
9273    ) -> Frame {
9274        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9275    }
9276
9277    fn hello_frame_with_nonce(
9278        module_id: &str,
9279        protocol_ver: u8,
9280        corr: u64,
9281        launch_nonce: Option<&str>,
9282    ) -> Frame {
9283        hello_frame_full(
9284            module_id,
9285            protocol_ver,
9286            corr,
9287            None,
9288            launch_nonce.map(ToOwned::to_owned),
9289        )
9290    }
9291
9292    fn hello_frame_full(
9293        module_id: &str,
9294        protocol_ver: u8,
9295        corr: u64,
9296        control_ops: Option<Vec<String>>,
9297        launch_nonce: Option<String>,
9298    ) -> Frame {
9299        let body = serde_json::to_vec(&ModuleHelloBody {
9300            manifest: manifest(module_id, protocol_ver),
9301            protocol_ver,
9302            control_ops,
9303            launch_nonce,
9304        })
9305        .unwrap();
9306        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9307    }
9308
9309    fn hello_frame_with_rich_ui(
9310        module_id: &str,
9311        protocol_ver: u8,
9312        corr: u64,
9313        launch_nonce: Option<&str>,
9314        rich_ui: Value,
9315    ) -> Frame {
9316        let mut body = serde_json::to_value(ModuleHelloBody {
9317            manifest: manifest(module_id, protocol_ver),
9318            protocol_ver,
9319            control_ops: None,
9320            launch_nonce: launch_nonce.map(str::to_string),
9321        })
9322        .unwrap();
9323        body["manifest"]["rich_ui"] = rich_ui;
9324        Frame::build(
9325            FrameType::Hello,
9326            control_flags(),
9327            0,
9328            0,
9329            corr,
9330            serde_json::to_vec(&body).unwrap(),
9331        )
9332        .unwrap()
9333    }
9334
9335    fn non_routable_hello_frame_with_control_ops(
9336        module_id: &str,
9337        corr: u64,
9338        control_ops: Option<Vec<String>>,
9339    ) -> Frame {
9340        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9341        manifest.provides.clear();
9342        let body = serde_json::to_vec(&ModuleHelloBody {
9343            manifest,
9344            protocol_ver: PROTOCOL_VERSION,
9345            control_ops,
9346            launch_nonce: None,
9347        })
9348        .unwrap();
9349        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9350    }
9351
9352    fn capability_grammar_hello_frame(
9353        capabilities: Value,
9354        runtime_computed: Option<Value>,
9355        corr: u64,
9356    ) -> Frame {
9357        let mut body = serde_json::to_value(ModuleHelloBody {
9358            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9359            protocol_ver: PROTOCOL_VERSION,
9360            control_ops: None,
9361            launch_nonce: None,
9362        })
9363        .expect("HELLO body serializes");
9364        body["manifest"]["capabilities"] = capabilities;
9365        if let Some(runtime_computed) = runtime_computed {
9366            body["runtime_computed"] = runtime_computed;
9367        }
9368        Frame::build(
9369            FrameType::Hello,
9370            control_flags(),
9371            0,
9372            0,
9373            corr,
9374            serde_json::to_vec(&body).expect("HELLO body reserializes"),
9375        )
9376        .expect("HELLO frame builds")
9377    }
9378
9379    fn channel_request(channel: u16, corr: u64) -> Frame {
9380        Frame::build(
9381            FrameType::Request,
9382            Flags::new(true, Priority::Interactive, false),
9383            channel,
9384            0,
9385            corr,
9386            b"opaque".to_vec(),
9387        )
9388        .unwrap()
9389    }
9390
9391    fn route_ctx(
9392        connection_id: ConnectionId,
9393    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9394        let (tx, rx) = mpsc::channel(8);
9395        (
9396            RouteCtx {
9397                connection_id,
9398                egress: FrameSink::new(tx),
9399            },
9400            rx,
9401        )
9402    }
9403
9404    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9405        serde_json::from_slice(&frame.body).unwrap()
9406    }
9407
9408    /// Register a module over a connection that has a sink and return the
9409    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
9410    /// module's own sink rather than returning it as a reply, so the ack is
9411    /// taken off `rx` here and whatever the test reads next is what followed it.
9412    async fn hello_via_sink(
9413        handler: &ControlHandler,
9414        ctx: &RouteCtx,
9415        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9416        hello: Frame,
9417    ) -> Frame {
9418        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9419        assert!(
9420            replies.is_empty(),
9421            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9422        );
9423        let ack = rx
9424            .try_recv()
9425            .expect("HELLO_ACK is queued on the module sink")
9426            .frame;
9427        assert_eq!(ack.header.ty, FrameType::HelloAck);
9428        ack
9429    }
9430
9431    fn parse_error(frame: &Frame) -> Value {
9432        serde_json::from_slice(&frame.body).unwrap()
9433    }
9434
9435    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9436        serde_json::from_slice(&frame.body).unwrap()
9437    }
9438
9439    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9440        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9441            route_channel,
9442            route_epoch: 0,
9443            kind,
9444        })
9445        .unwrap();
9446        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9447    }
9448
9449    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9450        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9451            module_id: module_id.to_string(),
9452        })
9453        .unwrap();
9454        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9455    }
9456
9457    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9458        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9459    }
9460
9461    // Exercise retired opener fields without putting them back in BindIdentity.
9462    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9463        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9464        let mut frame = route_open_frame(corr, module_id, root);
9465        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9466        body["identity"]["project_root"] = serde_json::json!(path.as_path());
9467        frame.body = serde_json::to_vec(&body).unwrap();
9468        frame
9469    }
9470
9471    fn route_open_frame_with_consumer_capabilities(
9472        corr: u64,
9473        module_id: &str,
9474        _project_root: ScratchDir,
9475        consumer_capabilities: Option<Vec<String>>,
9476    ) -> Frame {
9477        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9478            busy_refusals: None,
9479            target: RouteTarget::ToolProvider {
9480                module_id: module_id.to_string(),
9481            },
9482            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9483            consumer_identity: None,
9484            consumer_capabilities,
9485            role_versions: None,
9486            admission_facts: None,
9487            scope: None,
9488        })
9489        .unwrap();
9490        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9491    }
9492
9493    fn route_open_frame_with_role_versions(
9494        corr: u64,
9495        module_id: &str,
9496        _project_root: ScratchDir,
9497        role_versions: Option<BTreeMap<String, String>>,
9498    ) -> Frame {
9499        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9500            busy_refusals: None,
9501            target: RouteTarget::ToolProvider {
9502                module_id: module_id.to_string(),
9503            },
9504            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9505            consumer_identity: None,
9506            consumer_capabilities: None,
9507            role_versions,
9508            admission_facts: None,
9509            scope: None,
9510        })
9511        .unwrap();
9512        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9513    }
9514
9515    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9516        entries
9517            .iter()
9518            .map(|(role, version)| (role.to_string(), version.to_string()))
9519            .collect()
9520    }
9521
9522    fn route_open_frame_with_admission_facts(
9523        corr: u64,
9524        module_id: &str,
9525        _project_root: ScratchDir,
9526        consumer_identity: Option<subc_control::ConsumerIdentity>,
9527        facts: Option<Value>,
9528    ) -> Frame {
9529        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9530            busy_refusals: None,
9531            target: RouteTarget::ToolProvider {
9532                module_id: module_id.to_string(),
9533            },
9534            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9535            consumer_identity,
9536            consumer_capabilities: None,
9537            role_versions: None,
9538            admission_facts: facts,
9539            scope: None,
9540        })
9541        .unwrap();
9542        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9543    }
9544
9545    #[derive(Clone, Default)]
9546    struct EventCapture {
9547        events: Arc<Mutex<Vec<CapturedEvent>>>,
9548    }
9549
9550    #[derive(Clone, Debug)]
9551    struct CapturedEvent {
9552        target: String,
9553        level: tracing::Level,
9554        fields: BTreeMap<String, String>,
9555    }
9556
9557    impl EventCapture {
9558        fn events(&self) -> Vec<CapturedEvent> {
9559            self.events.lock().unwrap().clone()
9560        }
9561    }
9562
9563    impl<S> Layer<S> for EventCapture
9564    where
9565        S: Subscriber,
9566    {
9567        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9568            let mut visitor = EventFieldVisitor::default();
9569            event.record(&mut visitor);
9570            self.events.lock().unwrap().push(CapturedEvent {
9571                target: event.metadata().target().to_string(),
9572                level: *event.metadata().level(),
9573                fields: visitor.fields,
9574            });
9575        }
9576    }
9577
9578    #[derive(Default)]
9579    struct EventFieldVisitor {
9580        fields: BTreeMap<String, String>,
9581    }
9582
9583    impl Visit for EventFieldVisitor {
9584        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9585            self.fields
9586                .insert(field.name().to_string(), format!("{value:?}"));
9587        }
9588    }
9589
9590    fn health_response(corr: u64, status: HealthStatus) -> Frame {
9591        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9592            status,
9593            detail: Some("warming".to_string()),
9594            metrics: Some(json!({"queue_depth": 3})),
9595        })
9596        .unwrap();
9597        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9598    }
9599
9600    fn route_bind_ack(corr: u64) -> Frame {
9601        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9602        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9603    }
9604
9605    fn unique_project_root(label: &str) -> ScratchDir {
9606        ScratchDir::new(label)
9607    }
9608
9609    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9610        match parse_route_poll(frame) {
9611            ClientControlResponse::RoutePoll {
9612                status: None,
9613                live: Some(live),
9614                ..
9615            } => assert_eq!(live, expected_live),
9616            other => panic!("unexpected route.poll response: {other:?}"),
9617        }
9618    }
9619
9620    fn bind_liveness_route(
9621        registry: &Registry,
9622        forwarding: &ForwardingTable,
9623        module_id: &str,
9624    ) -> (RouteCtx, u16, u32) {
9625        let module_connection = ConnectionId::new(101);
9626        let client_connection = ConnectionId::new(202);
9627        let registration = registry
9628            .register_with_control_ops(
9629                manifest(module_id, PROTOCOL_VERSION),
9630                PROTOCOL_VERSION,
9631                module_connection,
9632                module_baseline_control_ops(),
9633            )
9634            .unwrap();
9635        let (module_tx, _module_rx) = mpsc::channel(8);
9636        let endpoint = forwarding
9637            .register_module_connection(
9638                module_connection,
9639                module_id.to_string(),
9640                PROTOCOL_VERSION,
9641                manifest_concurrency(&registration.manifest),
9642                FrameSink::new(module_tx),
9643            )
9644            .unwrap();
9645        let (client_ctx, _client_rx) = route_ctx(client_connection);
9646        let pending = forwarding
9647            .begin_route_bind_relay_for_test(
9648                client_connection,
9649                client_ctx.egress.clone(),
9650                1,
9651                module_id,
9652            )
9653            .unwrap();
9654        assert_eq!(pending.endpoint, endpoint);
9655        let route_channel = pending.client_channel;
9656        let route_epoch = pending.client_epoch;
9657        forwarding
9658            .complete_pending_relay(
9659                module_connection,
9660                pending.corr,
9661                RouteBindRelayOutcome::Accepted,
9662            )
9663            .unwrap();
9664        (client_ctx, route_channel, route_epoch)
9665    }
9666
9667    struct FakeProcessLiveness {
9668        live: Option<bool>,
9669    }
9670
9671    impl ModuleProcessLiveness for FakeProcessLiveness {
9672        fn process_live(&self, _module_id: &str) -> Option<bool> {
9673            self.live
9674        }
9675    }
9676
9677    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9678    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9679    {
9680        let registry = Arc::new(Registry::default());
9681        let supervisor_handle = SupervisorHandle::new();
9682        let supervisor = Supervisor::new_for_test(
9683            Arc::clone(&registry),
9684            RestartPolicy::new(1, Duration::from_millis(10)),
9685        )
9686        .with_handle(supervisor_handle.clone());
9687        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9688        let module = supervisor
9689            .spawn(ModuleSpec {
9690                carry_containment: false,
9691                module_id: "stderr-tail-wire".to_string(),
9692                program: fake_aft_stub_path(),
9693                args: Vec::new(),
9694                env: vec![
9695                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9696                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9697                ],
9698                reserved: false,
9699                reserved_prefixes: Vec::new(),
9700                protocol: ModuleProtocol::Subc,
9701                overlap: Default::default(),
9702            })
9703            .unwrap();
9704
9705        let deadline = Instant::now() + Duration::from_secs(5);
9706        loop {
9707            let tail = module.stderr_tail(None, None);
9708            if tail
9709                .entries
9710                .iter()
9711                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9712                && tail.entries.iter().any(|entry| {
9713                    matches!(
9714                        entry,
9715                        TailEntry::Line {
9716                            truncated: true,
9717                            ..
9718                        }
9719                    )
9720                })
9721            {
9722                break;
9723            }
9724            assert!(
9725                Instant::now() < deadline,
9726                "module did not produce a truncated line and restart boundary: {tail:?}"
9727            );
9728            sleep(Duration::from_millis(10)).await;
9729        }
9730
9731        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9732        let request = ClientControlRequest::SupervisorStderrTail {
9733            module_id: "stderr-tail-wire".to_string(),
9734            max_lines: None,
9735            max_bytes: None,
9736        };
9737        let frame = Frame::build(
9738            FrameType::Request,
9739            control_flags(),
9740            0,
9741            0,
9742            1,
9743            serde_json::to_vec(&request).unwrap(),
9744        )
9745        .unwrap();
9746        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9747        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9748        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9749            serde_json::from_slice(&responses[0].body).unwrap()
9750        else {
9751            panic!("expected supervisor.stderr_tail response");
9752        };
9753
9754        assert!(
9755            tail.entries
9756                .iter()
9757                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9758            "the control response lost the restart boundary"
9759        );
9760        let Some(StderrTailEntry::Line {
9761            text,
9762            truncated,
9763            at_ms,
9764        }) = tail.entries.iter().find(|entry| {
9765            matches!(
9766                entry,
9767                StderrTailEntry::Line {
9768                    truncated: true,
9769                    ..
9770                }
9771            )
9772        })
9773        else {
9774            panic!("the control response lost the truncated line");
9775        };
9776        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9777        assert!(*truncated);
9778        assert!(
9779            at_ms.is_some(),
9780            "the control response lost the line's capture time"
9781        );
9782    }
9783
9784    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9785    /// read done on the worker thread would stall every other task until it
9786    /// finished; the read must run off the worker so this test's own task keeps
9787    /// running while the read is paused.
9788    #[tokio::test(flavor = "current_thread")]
9789    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9790        let dir = ScratchDir::new("terminals-off-worker");
9791        let journal_path = dir.join("terminals.jsonl");
9792        let registry = Arc::new(Registry::default());
9793        let supervisor_handle = SupervisorHandle::new();
9794        let supervisor =
9795            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9796                .with_handle(supervisor_handle.clone())
9797                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9798        let module = supervisor
9799            .spawn(ModuleSpec {
9800                carry_containment: false,
9801                module_id: "terminal-off-worker".to_string(),
9802                program: fake_aft_stub_path(),
9803                args: Vec::new(),
9804                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9805                reserved: false,
9806                reserved_prefixes: Vec::new(),
9807                protocol: ModuleProtocol::Subc,
9808                overlap: Default::default(),
9809            })
9810            .unwrap();
9811        let deadline = Instant::now() + Duration::from_secs(5);
9812        while module.terminal_history().entries.len() != 2 {
9813            assert!(Instant::now() < deadline, "module did not record two exits");
9814            sleep(Duration::from_millis(10)).await;
9815        }
9816
9817        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9818        let handler =
9819            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9820        let frame = Frame::build(
9821            FrameType::Request,
9822            control_flags(),
9823            0,
9824            0,
9825            1,
9826            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9827                module_id: "terminal-off-worker".to_string(),
9828            })
9829            .unwrap(),
9830        )
9831        .unwrap();
9832        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9833        let spawned_at = std::time::Instant::now();
9834        let read = tokio::spawn({
9835            let handler = Arc::clone(&handler);
9836            async move { handler.handle_control_frame(&ctx, frame).await }
9837        });
9838        // Waiting for the pause from a blocking thread keeps this task pending,
9839        // so the runtime's single worker is free to run the read task.
9840        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9841            .await
9842            .unwrap()
9843            .expect("the history read reached its pause");
9844        let elapsed = spawned_at.elapsed();
9845        assert!(
9846            elapsed < Duration::from_secs(2) && !read.is_finished(),
9847            "this task could not run while the history read was paused \
9848             (resumed after {elapsed:?}, read finished: {})",
9849            read.is_finished()
9850        );
9851
9852        drop(release);
9853        let responses = read.await.unwrap().unwrap();
9854        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9855        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9856            panic!("expected supervisor.terminals response");
9857        };
9858        assert_eq!(terminals.entries.len(), 2);
9859        assert_eq!(terminals.journal_skipped_lines, 0);
9860        assert_eq!(terminals.journal_read_errors, 0);
9861    }
9862
9863    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9864    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9865        let registry = Arc::new(Registry::default());
9866        let supervisor_handle = SupervisorHandle::new();
9867        let supervisor =
9868            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9869                .with_handle(supervisor_handle.clone());
9870        let module = supervisor
9871            .spawn(ModuleSpec {
9872                carry_containment: false,
9873                module_id: "terminal-golden".to_string(),
9874                program: fake_aft_stub_path(),
9875                args: Vec::new(),
9876                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9877                reserved: false,
9878                reserved_prefixes: Vec::new(),
9879                protocol: ModuleProtocol::Subc,
9880                overlap: Default::default(),
9881            })
9882            .unwrap();
9883
9884        let deadline = Instant::now() + Duration::from_secs(5);
9885        while module.terminal_history().entries.len() != 2 {
9886            assert!(
9887                Instant::now() < deadline,
9888                "module did not retain two terminal exits: {:?}",
9889                module.terminal_history()
9890            );
9891            sleep(Duration::from_millis(10)).await;
9892        }
9893
9894        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9895        let request = ClientControlRequest::SupervisorTerminals {
9896            module_id: "terminal-golden".to_string(),
9897        };
9898        let frame = Frame::build(
9899            FrameType::Request,
9900            control_flags(),
9901            0,
9902            0,
9903            1,
9904            serde_json::to_vec(&request).unwrap(),
9905        )
9906        .unwrap();
9907        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9908        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9909        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9910        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9911            panic!("expected supervisor.terminals response");
9912        };
9913        assert_eq!(terminals.entries.len(), 2);
9914        assert_eq!(terminals.dropped, 0);
9915
9916        let mut rendered = serde_json::to_value(response).unwrap();
9917        // Wall-clock fields are the observation contract, but not stable fixture
9918        // bytes; normalize only them after the real handler has shaped the response.
9919        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9920        for (index, entry) in rendered["entries"]
9921            .as_array_mut()
9922            .expect("terminal response entries array")
9923            .iter_mut()
9924            .enumerate()
9925        {
9926            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9927        }
9928
9929        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9930            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9931        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9932        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9933            std::fs::write(&golden_path, &serialized).unwrap();
9934        }
9935        let expected: Value =
9936            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9937        assert_eq!(rendered, expected);
9938    }
9939
9940    #[test]
9941    fn hello_registers_manifest_and_returns_ack() {
9942        let registry = Arc::new(Registry::default());
9943        let handler = ControlHandler::new(Arc::clone(&registry));
9944        let conn = ConnectionId::new(1);
9945
9946        let responses = handler
9947            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9948            .unwrap();
9949
9950        assert_eq!(responses.len(), 1);
9951        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9952        assert_eq!(responses[0].header.channel, 0);
9953        assert_eq!(responses[0].header.corr, 7);
9954        let ack = parse_ack(&responses[0]);
9955        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9956        assert!(ack
9957            .subc_capabilities
9958            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9959        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9960        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9961        assert!(ack
9962            .subc_ops
9963            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9964        assert!(ack
9965            .subc_ops
9966            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9967
9968        let registration = registry.get_module("aft").unwrap().unwrap();
9969        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9970        assert_eq!(registration.state, ChannelState::Active);
9971        assert_eq!(registration.connection_id, conn);
9972        assert_eq!(registration.control_ops, module_baseline_control_ops());
9973        assert!(registration.manifest.display_name.is_none());
9974    }
9975
9976    #[test]
9977    fn hello_display_name_refuses_invalid_manifest_by_field() {
9978        let registry = Arc::new(Registry::default());
9979        let handler = ControlHandler::new(Arc::clone(&registry));
9980        for (index, invalid) in [
9981            json!(""),
9982            json!("a".repeat(41)),
9983            json!(" Routing"),
9984            json!("a\nb"),
9985            json!("a\u{202e}b"),
9986            json!(7),
9987        ]
9988        .into_iter()
9989        .enumerate()
9990        {
9991            let module_id = format!("bad-display-name-{index}");
9992            let mut body = serde_json::to_value(ModuleHelloBody {
9993                manifest: manifest(&module_id, PROTOCOL_VERSION),
9994                protocol_ver: PROTOCOL_VERSION,
9995                control_ops: None,
9996                launch_nonce: None,
9997            })
9998            .unwrap();
9999            body["manifest"]["display_name"] = invalid;
10000            let hello = Frame::build(
10001                FrameType::Hello,
10002                control_flags(),
10003                0,
10004                0,
10005                7,
10006                serde_json::to_vec(&body).unwrap(),
10007            )
10008            .unwrap();
10009            let response = handler
10010                .handle_control(ConnectionId::new(100 + index as u64), hello)
10011                .unwrap();
10012            assert_eq!(response[0].header.ty, FrameType::Error);
10013            let error = parse_error(&response[0]);
10014            assert_eq!(error["code"], "invalid_manifest");
10015            assert!(error["message"].as_str().unwrap().contains("display_name"));
10016            assert!(registry.get_module(&module_id).unwrap().is_none());
10017        }
10018    }
10019
10020    #[test]
10021    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10022        let invalid_identifiers = [
10023            ("case_change", "credentials-Provider/v1"),
10024            ("leading_zero", "credentials-provider/v01"),
10025            ("trailing_hyphen", "credentials-provider-/v1"),
10026            ("consecutive_hyphens", "credentials--provider/v1"),
10027            ("uppercase", "Credentials-provider/v1"),
10028            ("missing_v", "credentials-provider/1"),
10029            ("whitespace", "credentials provider/v1"),
10030            ("zero_version", "credentials-provider/v0"),
10031            ("out_of_range_version", "credentials-provider/v4294967296"),
10032            (
10033                "overlength_name",
10034                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10035            ),
10036        ];
10037        let mut cases = invalid_identifiers
10038            .into_iter()
10039            .map(|(name, identifier)| {
10040                (
10041                    format!("identifier_{name}"),
10042                    "capabilities.provides[0]".to_string(),
10043                    identifier.to_string(),
10044                    json!({ "provides": [identifier] }),
10045                    None,
10046                )
10047            })
10048            .collect::<Vec<_>>();
10049        cases.extend([
10050            (
10051                "unknown_need".to_string(),
10052                "capabilities.requires[0].need".to_string(),
10053                "deferred".to_string(),
10054                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10055                None,
10056            ),
10057            (
10058                "duplicate_provides".to_string(),
10059                "capabilities.provides[1]".to_string(),
10060                "credentials-provider/v1".to_string(),
10061                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10062                None,
10063            ),
10064            (
10065                "duplicate_must_never_reach".to_string(),
10066                "capabilities.must_never_reach[1]".to_string(),
10067                "credentials-provider/v1".to_string(),
10068                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10069                None,
10070            ),
10071            (
10072                "duplicate_requires_same_need".to_string(),
10073                "capabilities.requires[1]".to_string(),
10074                "credentials-provider/v1".to_string(),
10075                json!({ "requires": [
10076                    { "capability": "credentials-provider/v1", "need": "required" },
10077                    { "capability": "credentials-provider/v1", "need": "required" }
10078                ] }),
10079                None,
10080            ),
10081            (
10082                "duplicate_requires_conflicting_need".to_string(),
10083                "capabilities.requires[1]".to_string(),
10084                "credentials-provider/v1".to_string(),
10085                json!({ "requires": [
10086                    { "capability": "credentials-provider/v1", "need": "required" },
10087                    { "capability": "credentials-provider/v1", "need": "optional" }
10088                ] }),
10089                None,
10090            ),
10091            (
10092                "capabilities_root_pointer".to_string(),
10093                "runtime_computed[0]".to_string(),
10094                "/capabilities".to_string(),
10095                json!({}),
10096                Some(json!(["/capabilities"])),
10097            ),
10098            (
10099                "capabilities_descendant_pointer".to_string(),
10100                "runtime_computed[0]".to_string(),
10101                "/capabilities/provides".to_string(),
10102                json!({}),
10103                Some(json!(["/capabilities/provides"])),
10104            ),
10105            (
10106                "malformed_pointer_without_leading_slash".to_string(),
10107                "runtime_computed[0]".to_string(),
10108                "capabilities".to_string(),
10109                json!({}),
10110                Some(json!(["capabilities"])),
10111            ),
10112            (
10113                "malformed_pointer_escape".to_string(),
10114                "runtime_computed[0]".to_string(),
10115                "/roles/~2/tools".to_string(),
10116                json!({}),
10117                Some(json!(["/roles/~2/tools"])),
10118            ),
10119            (
10120                "unknown_capabilities_field".to_string(),
10121                "capabilities.future".to_string(),
10122                "<array>".to_string(),
10123                json!({ "future": [] }),
10124                None,
10125            ),
10126        ]);
10127
10128        for (index, (name, field, value, capabilities, runtime_computed)) in
10129            cases.into_iter().enumerate()
10130        {
10131            let registry = Arc::new(Registry::default());
10132            let handler = ControlHandler::new(Arc::clone(&registry));
10133            let response = handler
10134                .handle_control(
10135                    ConnectionId::new((index + 1) as u64),
10136                    capability_grammar_hello_frame(
10137                        capabilities,
10138                        runtime_computed,
10139                        index as u64 + 1,
10140                    ),
10141                )
10142                .expect("invalid HELLO returns a refusal");
10143
10144            assert_eq!(response.len(), 1, "{name} must emit one refusal");
10145            let error = parse_error(&response[0]);
10146            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10147            let message = error["message"]
10148                .as_str()
10149                .expect("error message is a string");
10150            assert!(
10151                message.contains(&field),
10152                "{name}: field missing from {message}"
10153            );
10154            assert!(
10155                message.contains(&value),
10156                "{name}: value missing from {message}"
10157            );
10158            assert_eq!(
10159                registry
10160                    .active_registration_count()
10161                    .expect("registry reads"),
10162                0,
10163                "{name}: refused HELLO must not create a catalog entry"
10164            );
10165        }
10166    }
10167
10168    #[test]
10169    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10170        let registry = Arc::new(Registry::default());
10171        let handler = ControlHandler::new(Arc::clone(&registry));
10172        let capabilities = json!({
10173            "provides": ["credentials-provider/v1"],
10174            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10175            "must_never_reach": ["federation-transport/v1"]
10176        });
10177        let response = handler
10178            .handle_control(
10179                ConnectionId::new(99),
10180                capability_grammar_hello_frame(
10181                    capabilities.clone(),
10182                    Some(json!(["/roles/0/tools"])),
10183                    99,
10184                ),
10185            )
10186            .expect("valid HELLO registers");
10187        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10188
10189        let request = Frame::build(
10190            FrameType::Request,
10191            control_flags(),
10192            0,
10193            0,
10194            100,
10195            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10196                .expect("catalog request serializes"),
10197        )
10198        .expect("catalog request frame builds");
10199        let response = handler
10200            .handle_catalog_list(request, None)
10201            .expect("catalog list succeeds");
10202        let ClientControlResponse::CatalogList { modules, .. } =
10203            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10204        else {
10205            panic!("catalog request must return catalog.list");
10206        };
10207        assert_eq!(modules.len(), 1);
10208        assert_eq!(
10209            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10210            capabilities
10211        );
10212    }
10213
10214    #[test]
10215    fn catalog_list_mirrors_management_operation_description() {
10216        let registry = Arc::new(Registry::default());
10217        let handler = ControlHandler::new(Arc::clone(&registry));
10218        let description = "List managed records and return their identifiers and metadata.";
10219        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10220        manifest.provides = vec![ProviderRole::ManagementSurface {
10221            operations: vec![ManagementOperation {
10222                name: "records.list".to_string(),
10223                kind: ManagementOperationKind::Query,
10224                description: Some(description.to_string()),
10225            }],
10226            config_schema: json!({"type": "object"}),
10227            observability: vec![ObservabilitySurface {
10228                name: "records.stats".to_string(),
10229                kind: ObservabilityKind::Snapshot,
10230            }],
10231            identity_scope: vec![IdentityScope::Project],
10232            concurrency: Concurrency::ModuleManaged,
10233        }];
10234        registry
10235            .register_with_control_ops(
10236                manifest,
10237                PROTOCOL_VERSION,
10238                ConnectionId::new(99),
10239                Vec::new(),
10240            )
10241            .expect("described management manifest registers");
10242
10243        let request = Frame::build(
10244            FrameType::Request,
10245            control_flags(),
10246            0,
10247            0,
10248            100,
10249            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10250                .expect("catalog request serializes"),
10251        )
10252        .expect("catalog request frame builds");
10253        let response = handler
10254            .handle_catalog_list(request, None)
10255            .expect("catalog list succeeds");
10256        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10257        assert_eq!(
10258            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10259            "catalog.list must preserve the declared operation description verbatim"
10260        );
10261    }
10262
10263    #[test]
10264    fn catalog_list_mirrors_registered_operation_delivery() {
10265        let registry = Arc::new(Registry::default());
10266        let handler = ControlHandler::new(Arc::clone(&registry));
10267        let delivery =
10268            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10269        let mut body = serde_json::to_value(ModuleHelloBody {
10270            manifest: manifest("watch-management", PROTOCOL_VERSION),
10271            protocol_ver: PROTOCOL_VERSION,
10272            control_ops: None,
10273            launch_nonce: None,
10274        })
10275        .unwrap();
10276        body["manifest"]["provides"] = json!([{
10277            "role": "management_surface",
10278            "operations": [
10279                {"name": "status", "kind": "query"},
10280                {"name": "future", "kind": "query"},
10281                {"name": "reset", "kind": "mutate"}
10282            ],
10283            "config_schema": {}, "observability": [], "identity_scope": []
10284        }]);
10285        body["manifest"]["operation_delivery"] = delivery.clone();
10286        let hello = Frame::build(
10287            FrameType::Hello,
10288            control_flags(),
10289            0,
10290            0,
10291            99,
10292            serde_json::to_vec(&body).unwrap(),
10293        )
10294        .unwrap();
10295        let response = handler
10296            .handle_control(ConnectionId::new(99), hello)
10297            .unwrap();
10298        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10299
10300        let (_, registrations) = registry.list_modules().unwrap();
10301        let registered = &registrations[0].manifest;
10302        assert!(registered.is_watch_operation("status"));
10303        assert!(!registered.is_watch_operation("future"));
10304        assert!(!registered.is_watch_operation("reset"));
10305        assert!(!registered.is_watch_operation("missing"));
10306
10307        let request = Frame::build(
10308            FrameType::Request,
10309            control_flags(),
10310            0,
10311            0,
10312            100,
10313            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10314        )
10315        .unwrap();
10316        let response = handler.handle_catalog_list(request, None).unwrap();
10317        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10318        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10319        let ClientControlResponse::CatalogList { modules, .. } =
10320            serde_json::from_slice(&response[0].body).unwrap()
10321        else {
10322            panic!("expected catalog.list");
10323        };
10324        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10325    }
10326
10327    #[test]
10328    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10329        let registry = Arc::new(Registry::default());
10330        let supervisor = SupervisorHandle::new();
10331        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10332        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10333        let descriptor = json!({
10334            "schema": "ckios.richui.manifest/1",
10335            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10336            "bytes": 4096,
10337            "chunks": 4
10338        });
10339        let connection = ConnectionId::new(201);
10340        let hello = handler
10341            .handle_control(
10342                connection,
10343                hello_frame_with_rich_ui(
10344                    "vault",
10345                    PROTOCOL_VERSION,
10346                    201,
10347                    Some("the-real-nonce"),
10348                    descriptor.clone(),
10349                ),
10350            )
10351            .unwrap();
10352        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10353
10354        let request =
10355            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10356        let provides = registry
10357            .get_module_by_connection(connection)
10358            .unwrap()
10359            .unwrap()
10360            .manifest
10361            .provides;
10362        handler
10363            .handle_catalog_update(connection, request, provides, None, Some(false))
10364            .unwrap();
10365
10366        let request = Frame::build(
10367            FrameType::Request,
10368            control_flags(),
10369            0,
10370            0,
10371            203,
10372            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10373        )
10374        .unwrap();
10375        let response = handler.handle_catalog_list(request, None).unwrap();
10376        let ClientControlResponse::CatalogList { modules, .. } =
10377            serde_json::from_slice(&response[0].body).unwrap()
10378        else {
10379            panic!("expected catalog.list");
10380        };
10381        assert_eq!(
10382            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10383            descriptor
10384        );
10385        assert!(modules[0].nonce_verified);
10386    }
10387
10388    fn hello_with_role_declarations(declarations: Value) -> Frame {
10389        let mut body = serde_json::to_value(ModuleHelloBody {
10390            manifest: manifest("role-test", PROTOCOL_VERSION),
10391            protocol_ver: PROTOCOL_VERSION,
10392            control_ops: None,
10393            launch_nonce: None,
10394        })
10395        .unwrap();
10396        body["manifest"]["role_declarations"] = declarations;
10397        Frame::build(
10398            FrameType::Hello,
10399            control_flags(),
10400            0,
10401            0,
10402            301,
10403            serde_json::to_vec(&body).unwrap(),
10404        )
10405        .unwrap()
10406    }
10407
10408    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10409        let request = Frame::build(
10410            FrameType::Request,
10411            control_flags(),
10412            0,
10413            0,
10414            302,
10415            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10416        )
10417        .unwrap();
10418        let response = handler.handle_catalog_list(request, None).unwrap();
10419        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10420        let ClientControlResponse::CatalogList { mut modules, .. } =
10421            serde_json::from_slice(&response[0].body).unwrap()
10422        else {
10423            panic!("expected catalog.list");
10424        };
10425        (raw["modules"][0].clone(), modules.remove(0))
10426    }
10427
10428    #[test]
10429    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10430        let registry = Arc::new(Registry::default());
10431        let handler = ControlHandler::new(Arc::clone(&registry));
10432        let declarations = json!({
10433            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10434            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10435        });
10436        let connection = ConnectionId::new(301);
10437        let response = handler
10438            .handle_control(
10439                connection,
10440                hello_with_role_declarations(declarations.clone()),
10441            )
10442            .unwrap();
10443        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10444        let registered = registry
10445            .get_module_by_connection(connection)
10446            .unwrap()
10447            .unwrap()
10448            .manifest;
10449        // Compared as JSON values, not bytes: object key order is not part of
10450        // the relay's promise, and workspace builds unify serde_json's
10451        // preserve_order feature, which changes the order a literal keeps.
10452        assert_eq!(
10453            serde_json::to_value(&registered.role_declarations).unwrap(),
10454            declarations
10455        );
10456        let (raw, entry) = role_declarations_catalog(&handler);
10457        assert_eq!(raw["role_declarations"], declarations);
10458        assert_eq!(entry.role_declarations, registered.role_declarations);
10459
10460        let request =
10461            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10462        handler
10463            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10464            .unwrap();
10465        let (raw, entry) = role_declarations_catalog(&handler);
10466        assert_eq!(raw["role_declarations"], declarations);
10467        assert_eq!(entry.role_declarations, registered.role_declarations);
10468    }
10469
10470    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10471        let registry = Arc::new(Registry::default());
10472        let handler = ControlHandler::new(Arc::clone(&registry));
10473        let capture = EventCapture::default();
10474        let _subscriber =
10475            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10476        let response = handler
10477            .handle_control(
10478                ConnectionId::new(301),
10479                hello_with_role_declarations(declarations),
10480            )
10481            .unwrap();
10482        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10483        assert!(registry
10484            .get_module_by_connection(ConnectionId::new(301))
10485            .unwrap()
10486            .unwrap()
10487            .manifest
10488            .role_declarations
10489            .is_empty());
10490        let (raw, entry) = role_declarations_catalog(&handler);
10491        assert!(raw.get("role_declarations").is_none());
10492        assert!(entry.role_declarations.is_empty());
10493        let warnings: Vec<_> = capture
10494            .events()
10495            .into_iter()
10496            .filter(|event| {
10497                event.fields.get("message").map(String::as_str)
10498                    == Some("dropping whole role_declarations map from HELLO")
10499            })
10500            .collect();
10501        assert_eq!(warnings.len(), 1);
10502        assert_eq!(warnings[0].level, tracing::Level::WARN);
10503        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10504        assert_eq!(warnings[0].fields["reason"], reason);
10505    }
10506
10507    #[test]
10508    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10509        for value in [
10510            json!(null),
10511            json!(false),
10512            json!(0),
10513            json!("opaque"),
10514            json!([]),
10515        ] {
10516            assert_hello_drops_role_declarations_with_warning(
10517                json!({"routing/v1": {}, "broken/v1": value}),
10518                "role_declarations values must be objects",
10519            );
10520        }
10521        assert_hello_drops_role_declarations_with_warning(
10522            json!([]),
10523            "role_declarations must be an object",
10524        );
10525    }
10526
10527    #[test]
10528    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10529        assert_hello_drops_role_declarations_with_warning(
10530            json!({"routing/v1": {}, "bad key": {}}),
10531            "role_declarations key must use capability-name grammar",
10532        );
10533    }
10534
10535    #[test]
10536    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10537        assert_hello_drops_role_declarations_with_warning(
10538            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10539            "role_declarations exceeds 16384 compact JSON bytes",
10540        );
10541    }
10542
10543    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10544        let mut body = serde_json::to_value(ModuleHelloBody {
10545            manifest: manifest("flow-ops", PROTOCOL_VERSION),
10546            protocol_ver: PROTOCOL_VERSION,
10547            control_ops: None,
10548            launch_nonce: None,
10549        })
10550        .unwrap();
10551        body["manifest"]["provides"] = json!([{
10552            "role": "management_surface",
10553            "operations": [
10554                {"name": "diagnostics.findings", "kind": "query"},
10555                {"name": "reset", "kind": "mutate"}
10556            ],
10557            "config_schema": {}, "observability": [], "identity_scope": []
10558        }]);
10559        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10560        if let Some(flow_operations) = flow_operations {
10561            body["manifest"]["flow_operations"] = flow_operations;
10562        }
10563        Frame::build(
10564            FrameType::Hello,
10565            control_flags(),
10566            0,
10567            0,
10568            401,
10569            serde_json::to_vec(&body).unwrap(),
10570        )
10571        .unwrap()
10572    }
10573
10574    #[test]
10575    fn catalog_list_mirrors_flow_operations_unchanged() {
10576        // Order is kept as declared (not sorted) and an empty list stays an
10577        // affirmative "none", distinct from an absent declaration.
10578        for (capability, flow_operations) in [
10579            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10580            ("call-modes/v1", json!(["diagnostics.findings"])),
10581            ("flow-scopes/v1", json!([])),
10582        ] {
10583            let registry = Arc::new(Registry::default());
10584            let handler = ControlHandler::new(Arc::clone(&registry));
10585            let connection = ConnectionId::new(401);
10586            let response = handler
10587                .handle_control(
10588                    connection,
10589                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10590                )
10591                .unwrap();
10592            assert_eq!(response[0].header.ty, FrameType::HelloAck);
10593            let registered = registry
10594                .get_module_by_connection(connection)
10595                .unwrap()
10596                .unwrap()
10597                .manifest;
10598            assert_eq!(
10599                serde_json::to_value(&registered.flow_operations).unwrap(),
10600                flow_operations
10601            );
10602            let (raw, entry) = role_declarations_catalog(&handler);
10603            assert_eq!(raw["flow_operations"], flow_operations);
10604            assert_eq!(entry.flow_operations, registered.flow_operations);
10605        }
10606
10607        let registry = Arc::new(Registry::default());
10608        let handler = ControlHandler::new(Arc::clone(&registry));
10609        let response = handler
10610            .handle_control(
10611                ConnectionId::new(401),
10612                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10613            )
10614            .unwrap();
10615        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10616        let (raw, entry) = role_declarations_catalog(&handler);
10617        assert!(raw.get("flow_operations").is_none());
10618        assert_eq!(entry.flow_operations, None);
10619    }
10620
10621    fn assert_hello_drops_flow_operations_with_warning(
10622        capabilities: Value,
10623        flow_operations: Value,
10624        reason: &str,
10625    ) {
10626        let registry = Arc::new(Registry::default());
10627        let handler = ControlHandler::new(Arc::clone(&registry));
10628        let capture = EventCapture::default();
10629        let _subscriber =
10630            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10631        let response = handler
10632            .handle_control(
10633                ConnectionId::new(401),
10634                hello_with_flow_operations(capabilities, Some(flow_operations)),
10635            )
10636            .unwrap();
10637        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10638        let registered = registry
10639            .get_module_by_connection(ConnectionId::new(401))
10640            .unwrap()
10641            .expect("the module still registers")
10642            .manifest;
10643        assert_eq!(registered.module_id, "flow-ops");
10644        assert_eq!(registered.flow_operations, None);
10645        let (raw, entry) = role_declarations_catalog(&handler);
10646        assert_eq!(raw["module_id"], "flow-ops");
10647        assert!(raw.get("flow_operations").is_none());
10648        assert_eq!(entry.flow_operations, None);
10649        let warnings: Vec<_> = capture
10650            .events()
10651            .into_iter()
10652            .filter(|event| {
10653                event.fields.get("message").map(String::as_str)
10654                    == Some("dropping flow_operations from HELLO")
10655            })
10656            .collect();
10657        assert_eq!(warnings.len(), 1);
10658        assert_eq!(warnings[0].level, tracing::Level::WARN);
10659        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10660        assert_eq!(warnings[0].fields["reason"], reason);
10661    }
10662
10663    #[test]
10664    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10665        assert_hello_drops_flow_operations_with_warning(
10666            json!(["flow-scopes/v1"]),
10667            json!(["diagnostics.findings", "missing"]),
10668            "flow_operations names 'missing', which is not a declared management operation",
10669        );
10670    }
10671
10672    #[test]
10673    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10674        assert_hello_drops_flow_operations_with_warning(
10675            json!([]),
10676            json!(["diagnostics.findings"]),
10677            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10678        );
10679        // A run-only capability does not admit flows either.
10680        assert_hello_drops_flow_operations_with_warning(
10681            json!(["agent-run-scopes/v1"]),
10682            json!([]),
10683            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10684        );
10685    }
10686
10687    #[test]
10688    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10689        assert_hello_drops_flow_operations_with_warning(
10690            json!(["flow-scopes/v1"]),
10691            json!(["reset", "reset"]),
10692            "flow_operations names 'reset' more than once",
10693        );
10694        assert_hello_drops_flow_operations_with_warning(
10695            json!(["flow-scopes/v1"]),
10696            json!(vec!["reset"; 257]),
10697            "flow_operations lists 257 names, more than 256",
10698        );
10699        assert_hello_drops_flow_operations_with_warning(
10700            json!(["flow-scopes/v1"]),
10701            json!("reset"),
10702            "flow_operations must be an array of operation names",
10703        );
10704    }
10705
10706    #[test]
10707    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10708        let registry = Arc::new(Registry::default());
10709        let handler = ControlHandler::new(registry.clone());
10710        let descriptor = json!({
10711            "schema": "ckios.richui.manifest/1",
10712            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10713            "bytes": 128,
10714            "chunks": 2
10715        });
10716        let malformed = [
10717            ("schema", json!("")),
10718            ("schema", json!("a".repeat(65))),
10719            ("schema", json!("Bad schema")),
10720            (
10721                "manifest_digest",
10722                json!(format!("sha256:{}", "A".repeat(64))),
10723            ),
10724            ("manifest_digest", json!("sha256:abc")),
10725            ("bytes", json!(0)),
10726            ("bytes", json!(16_777_217)),
10727            ("bytes", json!("128")),
10728            ("chunks", json!(0)),
10729            ("chunks", json!(1025)),
10730            ("chunks", json!("2")),
10731        ];
10732        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10733            let mut invalid = descriptor.clone();
10734            invalid[field] = invalid_value;
10735            let response = handler
10736                .handle_control(
10737                    ConnectionId::new(210 + index as u64),
10738                    hello_frame_with_rich_ui(
10739                        &format!("bad-rich-ui-{index}"),
10740                        PROTOCOL_VERSION,
10741                        210 + index as u64,
10742                        None,
10743                        invalid,
10744                    ),
10745                )
10746                .unwrap();
10747            assert_eq!(response[0].header.ty, FrameType::Error);
10748            let error = parse_error(&response[0]);
10749            assert_eq!(error["code"], "invalid_manifest");
10750            assert!(error["message"]
10751                .as_str()
10752                .unwrap()
10753                .contains(&format!("rich_ui.{field}")));
10754            assert!(registry
10755                .get_module(&format!("bad-rich-ui-{index}"))
10756                .unwrap()
10757                .is_none());
10758        }
10759    }
10760
10761    #[test]
10762    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10763        let registry = Arc::new(Registry::default());
10764        let handler = ControlHandler::new(Arc::clone(&registry));
10765        let connection = ConnectionId::new(219);
10766        registry
10767            .register_with_control_ops(
10768                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10769                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10770                        "bad schema",
10771                        format!("sha256:{}", "a".repeat(64)),
10772                        128,
10773                        2,
10774                    )))
10775                    .build(),
10776                PROTOCOL_VERSION,
10777                connection,
10778                Vec::new(),
10779            )
10780            .unwrap();
10781        let request =
10782            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10783        let response = handler
10784            .handle_catalog_update(connection, request, Vec::new(), None, None)
10785            .unwrap();
10786        assert_eq!(response[0].header.ty, FrameType::Error);
10787        let error = parse_error(&response[0]);
10788        assert_eq!(error["code"], "invalid_manifest");
10789        assert!(error["message"]
10790            .as_str()
10791            .unwrap()
10792            .contains("rich_ui.schema"));
10793    }
10794
10795    #[test]
10796    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10797        let registry = Arc::new(Registry::default());
10798        let supervisor = SupervisorHandle::new();
10799        supervisor.open_swap("provider", "candidate-nonce".to_string());
10800        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10801        let connection = ConnectionId::new(218);
10802        let hello = handler
10803            .handle_control(
10804                connection,
10805                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10806            )
10807            .unwrap();
10808        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10809        assert!(registry.list_modules().unwrap().1.is_empty());
10810
10811        registry.promote_candidate("provider").unwrap().unwrap();
10812        let request = Frame::build(
10813            FrameType::Request,
10814            control_flags(),
10815            0,
10816            0,
10817            219,
10818            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10819        )
10820        .unwrap();
10821        let response = handler.handle_catalog_list(request, None).unwrap();
10822        let ClientControlResponse::CatalogList { modules, .. } =
10823            serde_json::from_slice(&response[0].body).unwrap()
10824        else {
10825            panic!("expected catalog.list");
10826        };
10827        assert_eq!(modules.len(), 1);
10828        assert!(modules[0].nonce_verified);
10829    }
10830
10831    #[test]
10832    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10833        for provenance_json in [
10834            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10835            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10836        ] {
10837            let registry = Arc::new(Registry::default());
10838            let handler = ControlHandler::new(registry);
10839            let hello = Frame::build(
10840                FrameType::Hello,
10841                control_flags(),
10842                0,
10843                0,
10844                220,
10845                serde_json::to_vec(&ModuleHelloBody {
10846                    manifest: ModuleManifest::builder("built", "0.1.0")
10847                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10848                        .build(),
10849                    protocol_ver: PROTOCOL_VERSION,
10850                    control_ops: None,
10851                    launch_nonce: None,
10852                })
10853                .unwrap(),
10854            )
10855            .unwrap();
10856            assert_eq!(
10857                handler
10858                    .handle_control(ConnectionId::new(220), hello)
10859                    .unwrap()[0]
10860                    .header
10861                    .ty,
10862                FrameType::HelloAck
10863            );
10864            let request = Frame::build(
10865                FrameType::Request,
10866                control_flags(),
10867                0,
10868                0,
10869                221,
10870                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10871            )
10872            .unwrap();
10873            let response = handler.handle_catalog_list(request, None).unwrap();
10874            let ClientControlResponse::CatalogList { modules, .. } =
10875                serde_json::from_slice(&response[0].body).unwrap()
10876            else {
10877                panic!("expected catalog.list");
10878            };
10879            assert_eq!(modules.len(), 1);
10880            assert_eq!(
10881                serde_json::to_vec(&modules[0]).unwrap(),
10882                format!(
10883                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10884                )
10885                .as_bytes()
10886            );
10887        }
10888    }
10889
10890    #[test]
10891    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10892        let registry = Arc::new(Registry::default());
10893        let handler = ControlHandler::new(Arc::clone(&registry));
10894        let hello = Frame::build(
10895            FrameType::Hello,
10896            control_flags(),
10897            0,
10898            0,
10899            220,
10900            serde_json::to_vec(&ModuleHelloBody {
10901                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10902                protocol_ver: PROTOCOL_VERSION,
10903                control_ops: None,
10904                launch_nonce: Some("unverified".to_string()),
10905            })
10906            .unwrap(),
10907        )
10908        .unwrap();
10909        assert_eq!(
10910            handler
10911                .handle_control(ConnectionId::new(220), hello)
10912                .unwrap()[0]
10913                .header
10914                .ty,
10915            FrameType::HelloAck
10916        );
10917        let request = Frame::build(
10918            FrameType::Request,
10919            control_flags(),
10920            0,
10921            0,
10922            221,
10923            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10924        )
10925        .unwrap();
10926        let response = handler.handle_catalog_list(request, None).unwrap();
10927        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10928        assert!(wire["modules"][0].get("provenance").is_none());
10929        let ClientControlResponse::CatalogList { modules, .. } =
10930            serde_json::from_slice(&response[0].body).unwrap()
10931        else {
10932            panic!("expected catalog.list");
10933        };
10934        assert_eq!(
10935            serde_json::to_vec(&modules[0]).unwrap(),
10936            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10937        );
10938        assert!(!modules[0].nonce_verified);
10939        assert!(modules[0].provenance.is_none());
10940    }
10941
10942    #[test]
10943    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10944        let registry = Arc::new(Registry::default());
10945        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10946            [("vault".to_string(), true), ("squatter".to_string(), true)],
10947            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10948        );
10949        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10950        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10951            provides: vec!["credentials-provider/v1".to_string()],
10952            requires: Vec::new(),
10953            must_never_reach: Vec::new(),
10954        });
10955        let frame = Frame::build(
10956            FrameType::Hello,
10957            control_flags(),
10958            0,
10959            0,
10960            77,
10961            serde_json::to_vec(&ModuleHelloBody {
10962                manifest: squatter,
10963                protocol_ver: PROTOCOL_VERSION,
10964                control_ops: None,
10965                launch_nonce: None,
10966            })
10967            .expect("HELLO serializes"),
10968        )
10969        .expect("HELLO frame builds");
10970        let response = handler
10971            .handle_control(ConnectionId::new(77), frame)
10972            .expect("reserved claim receives a typed refusal");
10973        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10974        assert_eq!(
10975            registry
10976                .active_registration_count()
10977                .expect("registry reads"),
10978            0,
10979            "a reserved capability refusal must not leave a catalog entry"
10980        );
10981    }
10982
10983    #[test]
10984    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10985        for settlement in ["timeout", "inconclusive", "drop"] {
10986            let breakers = RouteBindBreakers::default();
10987            let RouteBindAdmission::Admitted {
10988                guard: mut old,
10989                probe: false,
10990            } = breakers.admit("prov")
10991            else {
10992                panic!("ordinary relay admitted")
10993            };
10994            let RouteBindAdmission::Admitted {
10995                guard: mut opener, ..
10996            } = breakers.admit("prov")
10997            else {
10998                panic!("second relay admitted")
10999            };
11000            assert!(
11001                !opener
11002                    .record_timeout(1, Duration::ZERO)
11003                    .unwrap()
11004                    .reopened_after_probe
11005            );
11006            let RouteBindAdmission::Admitted {
11007                guard: mut probe,
11008                probe: true,
11009            } = breakers.admit("prov")
11010            else {
11011                panic!("one cooldown probe admitted")
11012            };
11013            match settlement {
11014                "timeout" => assert!(
11015                    !old.record_timeout(1, Duration::ZERO)
11016                        .unwrap()
11017                        .reopened_after_probe
11018                ),
11019                "inconclusive" => old.record_inconclusive(),
11020                "drop" => drop(old),
11021                _ => unreachable!(),
11022            }
11023            assert!(
11024                matches!(
11025                    breakers.admit("prov"),
11026                    RouteBindAdmission::Refused {
11027                        probe_in_flight: true,
11028                        ..
11029                    }
11030                ),
11031                "{settlement} of a pre-open relay cannot release the real probe"
11032            );
11033            assert!(
11034                probe
11035                    .record_timeout(1, Duration::ZERO)
11036                    .unwrap()
11037                    .reopened_after_probe
11038            );
11039            assert!(matches!(
11040                breakers.admit("prov"),
11041                RouteBindAdmission::Admitted { probe: true, .. }
11042            ));
11043        }
11044        let breakers = RouteBindBreakers::default();
11045        let admit = || match breakers.admit("prov") {
11046            RouteBindAdmission::Admitted { guard, .. } => guard,
11047            _ => panic!("relay admitted"),
11048        };
11049        admit().record_timeout(1, Duration::ZERO);
11050        let mut old_probe = admit();
11051        breakers.reset_for_new_module_connection("prov");
11052        admit().record_timeout(1, Duration::ZERO);
11053        let _new_probe = admit();
11054        old_probe.record_inconclusive();
11055        assert!(matches!(
11056            breakers.admit("prov"),
11057            RouteBindAdmission::Refused {
11058                probe_in_flight: true,
11059                ..
11060            }
11061        ));
11062    }
11063
11064    #[tokio::test]
11065    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11066        for candidate in [false, true] {
11067            let registry = Arc::new(Registry::default());
11068            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
11069                [("vault".to_string(), true), ("squatter".to_string(), true)],
11070                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11071            );
11072            let conn = ConnectionId::new(77);
11073            let (ctx, mut rx) = route_ctx(conn);
11074            let initial = capability_manifest("squatter", &[], &[]);
11075            if candidate {
11076                registry
11077                    .register_candidate_with_control_ops(
11078                        initial.clone(),
11079                        PROTOCOL_VERSION,
11080                        conn,
11081                        module_baseline_control_ops(),
11082                    )
11083                    .unwrap();
11084                handler
11085                    .forwarding
11086                    .register_candidate_module_connection(
11087                        conn,
11088                        "squatter".to_string(),
11089                        PROTOCOL_VERSION,
11090                        manifest_concurrency(&initial),
11091                        ctx.egress.clone(),
11092                    )
11093                    .unwrap();
11094            } else {
11095                hello_via_sink(
11096                    &handler,
11097                    &ctx,
11098                    &mut rx,
11099                    hello_frame_with_manifest(initial.clone(), 1),
11100                )
11101                .await;
11102            }
11103            let response = handler
11104                .handle_control_frame(
11105                    &ctx,
11106                    catalog_update_with_capabilities_frame(
11107                        2,
11108                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
11109                            .capabilities
11110                            .unwrap(),
11111                    ),
11112                )
11113                .await
11114                .unwrap();
11115            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11116            assert_eq!(
11117                registry
11118                    .get_module_by_connection(conn)
11119                    .unwrap()
11120                    .unwrap()
11121                    .manifest,
11122                initial
11123            );
11124        }
11125    }
11126
11127    #[test]
11128    fn server_describe_surfaces_required_capability_verdict_fields() {
11129        let registry = Arc::new(Registry::default());
11130        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
11131            [
11132                ("consumer".to_string(), true),
11133                ("provider".to_string(), false),
11134            ],
11135            BTreeMap::new(),
11136        );
11137        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11138        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11139            provides: Vec::new(),
11140            requires: vec![subc_protocol::manifest::CapabilityRequirement {
11141                capability: "credentials-provider/v1".to_string(),
11142                need: subc_protocol::manifest::CapabilityNeed::Required,
11143            }],
11144            must_never_reach: Vec::new(),
11145        });
11146        let hello = Frame::build(
11147            FrameType::Hello,
11148            control_flags(),
11149            0,
11150            0,
11151            78,
11152            serde_json::to_vec(&ModuleHelloBody {
11153                manifest: consumer,
11154                protocol_ver: PROTOCOL_VERSION,
11155                control_ops: None,
11156                launch_nonce: None,
11157            })
11158            .expect("HELLO serializes"),
11159        )
11160        .expect("HELLO frame builds");
11161        handler
11162            .handle_control(ConnectionId::new(78), hello)
11163            .expect("consumer registers");
11164        let describe = Frame::build(
11165            FrameType::Request,
11166            control_flags(),
11167            0,
11168            0,
11169            79,
11170            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11171                .expect("request serializes"),
11172        )
11173        .expect("describe frame builds");
11174        let response = handler
11175            .handle_server_describe(describe)
11176            .expect("server.describe succeeds");
11177        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11178        let requirement = &rendered["capability_requirements"][0];
11179        assert_eq!(requirement["consumer"], "consumer");
11180        assert_eq!(requirement["verdict"], "never_provided");
11181        assert_eq!(requirement["episode_seq"], 1);
11182        assert_eq!(requirement["config_satisfiable"], false);
11183        assert_eq!(requirement["runtime_available"], false);
11184        assert!(requirement["detail"]
11185            .as_str()
11186            .expect("detail string")
11187            .contains("credentials-provider/v1"));
11188    }
11189
11190    /// Register a consumer that requires `project-identity/v1` and the provider
11191    /// of it, optionally mark daemon shutdown as begun, then deregister the
11192    /// provider as its connection closes. Returns the `capability.requirement`
11193    /// events that deregistration logged and the verdict the evaluator then
11194    /// holds for the requirement (the one `route.open` refusals read).
11195    fn deregister_required_provider(
11196        daemon_shutdown_begun: bool,
11197    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11198        const CAPABILITY: &str = "project-identity/v1";
11199        let registry = Arc::new(Registry::default());
11200        let daemon_shutdown = DaemonShutdownFlag::default();
11201        let handler = ControlHandler::new(Arc::clone(&registry))
11202            .with_capability_config(
11203                [
11204                    ("consumer".to_string(), true),
11205                    ("provider".to_string(), true),
11206                ],
11207                BTreeMap::new(),
11208            )
11209            .with_daemon_shutdown_flag(daemon_shutdown.clone());
11210        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11211        consumer.capabilities = Some(CapabilityDeclarations {
11212            provides: Vec::new(),
11213            requires: vec![subc_protocol::manifest::CapabilityRequirement {
11214                capability: CAPABILITY.to_string(),
11215                need: subc_protocol::manifest::CapabilityNeed::Required,
11216            }],
11217            must_never_reach: Vec::new(),
11218        });
11219        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11220        let provider_conn = ConnectionId::new(182);
11221        for (conn, manifest, corr) in [
11222            (ConnectionId::new(181), consumer, 181),
11223            (provider_conn, provider, 182),
11224        ] {
11225            let hello = handler
11226                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11227                .expect("module registers");
11228            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11229        }
11230        assert_eq!(
11231            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11232            Some(CapabilityVerdict::Provided),
11233            "the registered provider must satisfy the requirement first"
11234        );
11235        if daemon_shutdown_begun {
11236            daemon_shutdown.set_for_test();
11237        }
11238
11239        let capture = EventCapture::default();
11240        let _subscriber =
11241            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11242        handler
11243            .cleanup_connection(provider_conn)
11244            .expect("provider deregisters");
11245        let events = capture
11246            .events()
11247            .into_iter()
11248            .filter(|event| {
11249                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11250            })
11251            .collect();
11252        (
11253            events,
11254            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11255        )
11256    }
11257
11258    #[test]
11259    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11260        let (events, verdict) = deregister_required_provider(true);
11261        assert_eq!(
11262            verdict,
11263            Some(CapabilityVerdict::NeverProvided),
11264            "shutdown must not change the verdict route.open refusals read"
11265        );
11266        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11267        let event = &events[0];
11268        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11269        assert_eq!(
11270            event.fields.get("cause").map(String::as_str),
11271            Some("\"daemon_shutdown\""),
11272            "{event:?}"
11273        );
11274        assert_eq!(
11275            event.fields.get("verdict").map(String::as_str),
11276            Some("\"never_provided\""),
11277            "{event:?}"
11278        );
11279    }
11280
11281    #[test]
11282    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11283        let (events, verdict) = deregister_required_provider(false);
11284        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11285        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11286        let event = &events[0];
11287        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11288        assert_eq!(event.fields.get("cause"), None, "{event:?}");
11289        assert_eq!(
11290            event.fields.get("verdict").map(String::as_str),
11291            Some("\"never_provided\""),
11292            "{event:?}"
11293        );
11294    }
11295
11296    #[test]
11297    fn catalog_list_omits_capabilities_for_legacy_manifest() {
11298        let registry = Arc::new(Registry::default());
11299        let handler = ControlHandler::new(Arc::clone(&registry));
11300        let hello = handler
11301            .handle_control(
11302                ConnectionId::new(101),
11303                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11304            )
11305            .expect("legacy HELLO registers");
11306        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11307
11308        let request = Frame::build(
11309            FrameType::Request,
11310            control_flags(),
11311            0,
11312            0,
11313            102,
11314            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11315                .expect("catalog request serializes"),
11316        )
11317        .expect("catalog request frame builds");
11318        let response = handler
11319            .handle_catalog_list(request, None)
11320            .expect("catalog list succeeds");
11321        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11322        assert!(
11323            body["modules"][0].get("capabilities").is_none(),
11324            "legacy manifest must retain an absent capabilities field on catalog.list"
11325        );
11326    }
11327
11328    #[test]
11329    fn hello_ack_omits_storage_when_no_storage_config() {
11330        let registry = Arc::new(Registry::default());
11331        let handler = ControlHandler::new(Arc::clone(&registry));
11332        let responses = handler
11333            .handle_control(
11334                ConnectionId::new(1),
11335                hello_frame("aft", PROTOCOL_VERSION, 7),
11336            )
11337            .unwrap();
11338        let ack = parse_ack(&responses[0]);
11339        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11340        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11341    }
11342
11343    #[tokio::test]
11344    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11345        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11346        let registry = Arc::new(Registry::default());
11347        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
11348        let responses = handler
11349            .handle_control(
11350                ConnectionId::new(1),
11351                hello_frame("aft", PROTOCOL_VERSION, 7),
11352            )
11353            .unwrap();
11354        let ack = parse_ack(&responses[0]);
11355        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11356
11357        let described = handler
11358            .handle_control_frame(
11359                &route_ctx(ConnectionId::new(2)).0,
11360                Frame::build(
11361                    FrameType::Request,
11362                    control_flags(),
11363                    0,
11364                    0,
11365                    9,
11366                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11367                )
11368                .unwrap(),
11369            )
11370            .await
11371            .unwrap();
11372        let ClientControlResponse::ServerDescribe { machine_id, .. } =
11373            serde_json::from_slice(&described[0].body).unwrap()
11374        else {
11375            panic!("server.describe answered with another shape");
11376        };
11377        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11378    }
11379
11380    #[test]
11381    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11382        // With a central sqlite storage policy, each registering module gets its
11383        // own resolved descriptor in HELLO_ACK, keyed by its module id.
11384        let registry = Arc::new(Registry::default());
11385        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
11386            crate::daemon_config::StorageConfig::Sqlite {
11387                data_home: std::path::PathBuf::from("/data"),
11388            },
11389        ));
11390
11391        let responses = handler
11392            .handle_control(
11393                ConnectionId::new(1),
11394                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11395            )
11396            .unwrap();
11397        let ack = parse_ack(&responses[0]);
11398        assert_eq!(
11399            ack.storage,
11400            Some(serde_json::json!({
11401                "module_id": "alfonso-routing",
11402                "storage_namespace": "default",
11403                "isolation": { "kind": "module" },
11404                "backend": {
11405                    "backend": "sqlite",
11406                    "path": "/data/cortexkit/alfonso-routing/store.db"
11407                }
11408            })),
11409            "the delivered descriptor is the module's own sqlite store path"
11410        );
11411    }
11412
11413    #[test]
11414    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11415        let registry = Arc::new(Registry::default());
11416        let handler = ControlHandler::new(Arc::clone(&registry));
11417        let conn = ConnectionId::new(1);
11418        let responses = handler
11419            .handle_control(
11420                conn,
11421                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11422            )
11423            .unwrap();
11424        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11425        let registration = registry.get_module("aft").unwrap().unwrap();
11426        assert_eq!(registration.control_ops, module_baseline_control_ops());
11427
11428        let frame =
11429            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11430        assert!(handler
11431            .guard_module_control_op(&frame, "aft", "route.bind")
11432            .unwrap()
11433            .is_none());
11434        let error = handler
11435            .guard_module_control_op(&frame, "aft", "test.synthetic")
11436            .unwrap()
11437            .expect("synthetic ungranted op should be rejected");
11438        assert_eq!(error.header.ty, FrameType::Error);
11439        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11440    }
11441
11442    #[test]
11443    fn hello_control_ops_some_adds_optional_grants() {
11444        let registry = Arc::new(Registry::default());
11445        let handler = ControlHandler::new(Arc::clone(&registry));
11446        handler
11447            .handle_control(
11448                ConnectionId::new(1),
11449                hello_frame_with_control_ops(
11450                    "aft",
11451                    PROTOCOL_VERSION,
11452                    7,
11453                    Some(vec![
11454                        "future.synthetic".to_string(),
11455                        "route.bind".to_string(),
11456                    ]),
11457                ),
11458            )
11459            .unwrap();
11460        let registration = registry.get_module("aft").unwrap().unwrap();
11461        assert_eq!(
11462            registration.control_ops,
11463            vec![
11464                "route.bind".to_string(),
11465                "route.status".to_string(),
11466                "future.synthetic".to_string(),
11467            ]
11468        );
11469        let frame =
11470            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11471        assert!(handler
11472            .guard_module_control_op(&frame, "aft", "future.synthetic")
11473            .unwrap()
11474            .is_none());
11475    }
11476
11477    #[tokio::test]
11478    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11479        let registry = Arc::new(Registry::default());
11480        let forwarding = Arc::new(ForwardingTable::default());
11481        let handler =
11482            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11483        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11484        hello_via_sink(
11485            &handler,
11486            &module_ctx,
11487            &mut module_rx,
11488            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11489        )
11490        .await;
11491
11492        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11493        let responses = handler
11494            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11495            .await
11496            .unwrap();
11497        assert_eq!(responses.len(), 1);
11498        assert_eq!(responses[0].header.ty, FrameType::Error);
11499        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11500        assert!(module_rx.try_recv().is_err());
11501    }
11502
11503    #[tokio::test]
11504    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11505        let registry = Arc::new(Registry::default());
11506        let forwarding = Arc::new(ForwardingTable::default());
11507        let handler =
11508            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11509        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11510        hello_via_sink(
11511            &handler,
11512            &module_ctx,
11513            &mut module_rx,
11514            hello_frame_with_control_ops(
11515                "aft",
11516                PROTOCOL_VERSION,
11517                7,
11518                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11519            ),
11520        )
11521        .await;
11522
11523        let project_root = unique_project_root("demux");
11524        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11525        let route_handler = handler.clone();
11526        let route_task = tokio::spawn(async move {
11527            route_handler
11528                .handle_control_frame(
11529                    &route_client_ctx,
11530                    route_open_frame(100, "aft", project_root),
11531                )
11532                .await
11533                .unwrap()
11534        });
11535        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11536            .await
11537            .unwrap()
11538            .unwrap();
11539        assert!(matches!(
11540            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11541            ModuleControlRequest::RouteBind { .. }
11542        ));
11543
11544        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11545        let health_handler = handler.clone();
11546        let health_task = tokio::spawn(async move {
11547            health_handler
11548                .handle_control_frame(
11549                    &health_client_ctx,
11550                    supervisor_health_probe_frame(101, "aft"),
11551                )
11552                .await
11553                .unwrap()
11554        });
11555        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11556            .await
11557            .unwrap()
11558            .unwrap();
11559        assert_eq!(
11560            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11561            ModuleControlRequest::HealthCheck {}
11562        );
11563
11564        handler
11565            .handle_control_frame(
11566                &module_ctx,
11567                health_response(health_frame.header.corr, HealthStatus::Degraded),
11568            )
11569            .await
11570            .unwrap();
11571        let health_response = health_task.await.unwrap();
11572        assert_eq!(health_response.len(), 1);
11573        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11574            ClientControlResponse::SupervisorHealthProbe {
11575                module_id,
11576                status,
11577                detail,
11578                metrics,
11579            } => {
11580                assert_eq!(module_id, "aft");
11581                assert_eq!(status, HealthStatus::Degraded);
11582                assert_eq!(detail.as_deref(), Some("warming"));
11583                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11584            }
11585            other => panic!("unexpected health response: {other:?}"),
11586        }
11587
11588        handler
11589            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11590            .await
11591            .unwrap();
11592        let route_response = route_task.await.unwrap();
11593        assert!(route_response.is_empty());
11594        let published = route_client_rx.recv().await.unwrap();
11595        assert!(matches!(
11596            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11597            ClientControlResponse::RouteOpen { .. }
11598        ));
11599    }
11600
11601    /// Start one `route.open` on `client_connection` and return its still-running
11602    /// handler task together with the `route.bind` the module received for it.
11603    /// The handler blocks until the module answers, so it has to run as a task
11604    /// while the test drives the module side.
11605    async fn relay_route_open(
11606        handler: &ControlHandler,
11607        client_connection: ConnectionId,
11608        client_egress: &FrameSink,
11609        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11610        corr: u64,
11611        module_id: &str,
11612        project_root_label: &str,
11613    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11614        let ctx = RouteCtx {
11615            connection_id: client_connection,
11616            egress: client_egress.clone(),
11617        };
11618        let handler = handler.clone();
11619        let project_root = unique_project_root(project_root_label);
11620        let module_id = module_id.to_string();
11621        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11622        let task = tokio::spawn(async move {
11623            let _guard = tracing::dispatcher::set_default(&dispatch);
11624            handler
11625                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11626                .await
11627                .unwrap()
11628        });
11629        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11630            .await
11631            .expect("module receives the relayed route.bind")
11632            .expect("module egress is open");
11633        (task, bind.frame)
11634    }
11635
11636    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11637        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11638            ModuleControlRequest::RouteBind {
11639                route_channel,
11640                epoch,
11641                ..
11642            } => (route_channel, epoch),
11643            other => panic!("expected a route.bind request, got {other:?}"),
11644        }
11645    }
11646
11647    fn published_route(frame: &Frame) -> (u16, u32) {
11648        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11649            ClientControlResponse::RouteOpen {
11650                route_channel,
11651                route_epoch,
11652                ..
11653            } => (route_channel, route_epoch),
11654            other => panic!("expected a route.open response, got {other:?}"),
11655        }
11656    }
11657
11658    /// Reproduction of a production outage. A client had `route.open`s in
11659    /// flight to a module and was already marked closing -- its egress had refused a
11660    /// module frame, so the daemon asked its connection to end -- while its sink
11661    /// was still open. When the module acked those binds, the daemon refused to
11662    /// commit a route for a closing client, and that refusal was returned from
11663    /// the MODULE connection's frame handler, where a router error that has no
11664    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
11665    /// the supervisor correctly did not respawn a clean exit, and every seat lost
11666    /// its tools for hours -- one client's teardown took down a connection
11667    /// carrying ~170 other routes.
11668    ///
11669    /// The window is opened here by calling the production path that opens it
11670    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
11671    /// state that matters is "in `closing_connections`, sink still open, relay
11672    /// still pending", and it lasts only from the close request until the
11673    /// connection loop reacts to it; a socket-level test can flood a client into
11674    /// that escalation but cannot pin the module's ack inside the window. Closing
11675    /// the socket instead takes the other path entirely -- connection teardown
11676    /// removes the pending relay under the same lock, so the ack finds nothing.
11677    #[tokio::test]
11678    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11679        let registry = Arc::new(Registry::default());
11680        let forwarding = Arc::new(ForwardingTable::default());
11681        let handler =
11682            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11683
11684        let module_connection = ConnectionId::new(30);
11685        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11686        hello_via_sink(
11687            &handler,
11688            &module_ctx,
11689            &mut module_rx,
11690            hello_frame("aft", PROTOCOL_VERSION, 7),
11691        )
11692        .await;
11693
11694        let dying_client = ConnectionId::new(31);
11695        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11696
11697        // A published route on the dying client. The escalation below only marks
11698        // a connection closing for a route it has already published.
11699        let (first_task, first_bind) = relay_route_open(
11700            &handler,
11701            dying_client,
11702            &dying_ctx.egress,
11703            &mut module_rx,
11704            100,
11705            "aft",
11706            "closing-first",
11707        )
11708        .await;
11709        handler
11710            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11711            .await
11712            .unwrap();
11713        assert!(first_task.await.unwrap().is_empty());
11714        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11715
11716        // A second route.open from the same client, relayed and awaiting its ack.
11717        let (second_task, second_bind) = relay_route_open(
11718            &handler,
11719            dying_client,
11720            &dying_ctx.egress,
11721            &mut module_rx,
11722            101,
11723            "aft",
11724            "closing-second",
11725        )
11726        .await;
11727        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11728
11729        // The window: the client is closing, its sink is still open, and its
11730        // second bind is still pending.
11731        assert!(forwarding
11732            .escalate_client_delivery_failure(
11733                dying_client,
11734                first_channel,
11735                first_epoch,
11736                CloseReason::new(
11737                    "module_to_client_delivery_failed",
11738                    "client egress refused a module frame",
11739                ),
11740                crate::forwarding::UndeliveredFrame {
11741                    module_id: None,
11742                    sink: &dying_ctx.egress,
11743                },
11744            )
11745            .unwrap());
11746        assert!(!dying_ctx.egress.is_closed());
11747
11748        // The frame that used to end the module connection.
11749        let ack = handler
11750            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11751            .await;
11752        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11753        if module_loop_error.is_some() {
11754            // What the server's connection loop does with a router error that has
11755            // no ERROR-frame translation: end the connection, which releases the
11756            // module's registration and every route on it.
11757            handler.cleanup_connection(module_connection).unwrap();
11758        }
11759        // Read the module's next frame before opening the co-tenant's route, so
11760        // the GOODBYE assertion below is about THIS ack and not about later
11761        // traffic. `None` means the module was told nothing.
11762        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11763            .await
11764            .ok()
11765            .flatten();
11766
11767        // 1. The module connection is still registered.
11768        assert!(
11769            registry
11770                .get_module_by_connection(module_connection)
11771                .unwrap()
11772                .is_some(),
11773            "one client's closing connection ended the shared module connection: \
11774             {module_loop_error:?}"
11775        );
11776        // ...and still serving: another client can open and use a route on it.
11777        let cotenant = ConnectionId::new(32);
11778        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11779        let (cotenant_task, cotenant_bind) = relay_route_open(
11780            &handler,
11781            cotenant,
11782            &cotenant_ctx.egress,
11783            &mut module_rx,
11784            102,
11785            "aft",
11786            "closing-cotenant",
11787        )
11788        .await;
11789        handler
11790            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11791            .await
11792            .unwrap();
11793        assert!(cotenant_task.await.unwrap().is_empty());
11794        let (cotenant_channel, cotenant_epoch) =
11795            published_route(&cotenant_rx.recv().await.unwrap());
11796        assert!(matches!(
11797            forwarding
11798                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11799                .unwrap(),
11800            DataRoute::Client(DataRouteState::Bound(_))
11801        ));
11802
11803        // 2. The module was told to drop the binding it created for the route
11804        //    that will never be published.
11805        let goodbye = post_ack_module_frame
11806            .expect("module receives a GOODBYE for the abandoned route channel");
11807        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11808        assert_eq!(goodbye.header.channel, abandoned_channel);
11809        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11810
11811        // 3. The dying client received nothing: no route was ever published to
11812        //    it. Its route.open is answered as unavailable, which the connection
11813        //    loop would write to a socket that is already going away.
11814        assert!(dying_rx.try_recv().is_err());
11815        let second_response = second_task.await.unwrap();
11816        assert_eq!(second_response.len(), 1);
11817        assert_eq!(
11818            parse_error(&second_response[0])["code"],
11819            "target_unavailable"
11820        );
11821    }
11822
11823    /// One `catalog.list` as a client sees it: the generation and the listed
11824    /// module ids with their readiness.
11825    async fn catalog_snapshot(
11826        handler: &ControlHandler,
11827        client_ctx: &RouteCtx,
11828        corr: u64,
11829    ) -> (u64, Vec<(String, bool)>) {
11830        let body =
11831            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11832        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11833        let reply = handler
11834            .handle_control_frame(client_ctx, request)
11835            .await
11836            .unwrap()
11837            .pop()
11838            .unwrap();
11839        let ClientControlResponse::CatalogList {
11840            generation,
11841            modules,
11842            ..
11843        } = serde_json::from_slice(&reply.body).unwrap()
11844        else {
11845            panic!("not a catalog.list reply");
11846        };
11847        (
11848            generation,
11849            modules
11850                .into_iter()
11851                .map(|entry| (entry.module_id, entry.ready))
11852                .collect(),
11853        )
11854    }
11855
11856    /// A handler whose HELLO for module "aft" on connection 40 is parked
11857    /// between reserving its registry entry and installing its forwarding
11858    /// endpoint. Returns the handler, the two-party barrier (wait once to know
11859    /// the HELLO is parked, once more to release it), the HELLO's thread, and
11860    /// the module's context and receiver.
11861    fn hello_held_before_forwarding() -> (
11862        ControlHandler,
11863        Arc<std::sync::Barrier>,
11864        std::thread::JoinHandle<Vec<Frame>>,
11865        RouteCtx,
11866        mpsc::Receiver<crate::router::OutboundFrame>,
11867    ) {
11868        let hold = Arc::new(std::sync::Barrier::new(2));
11869        let handler = ControlHandler::with_forwarding(
11870            Arc::new(Registry::default()),
11871            Arc::new(ForwardingTable::default()),
11872        )
11873        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11874        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11875        let hello = {
11876            let handler = handler.clone();
11877            let module_ctx = module_ctx.clone();
11878            std::thread::spawn(move || {
11879                handler
11880                    .handle_hello(
11881                        module_ctx.connection_id,
11882                        Some(module_ctx.egress.clone()),
11883                        hello_frame("aft", PROTOCOL_VERSION, 7),
11884                    )
11885                    .unwrap()
11886            })
11887        };
11888        (handler, hold, hello, module_ctx, module_rx)
11889    }
11890
11891    /// While a HELLO has reserved its registry entry but its forwarding
11892    /// endpoint is not installed yet, `route.open` to the module can only be
11893    /// refused. `catalog.list` must not list the module in that window: a
11894    /// client that opens a route as soon as it sees a module listed and ready
11895    /// would be refused for a module the daemon had just advertised. The
11896    /// refusal in the window stays the retryable `target_unavailable`, never
11897    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
11898    /// Once the HELLO completes, the module is listed and a route to it opens.
11899    #[tokio::test]
11900    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11901        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11902        let client = ConnectionId::new(41);
11903        let (client_ctx, mut client_rx) = route_ctx(client);
11904
11905        hold.wait();
11906        // Parked: the registry holds "aft", forwarding has no endpoint for it.
11907        assert!(handler.registry.get_module("aft").unwrap().is_some());
11908        assert!(!handler
11909            .forwarding
11910            .has_live_module_connection("aft")
11911            .unwrap());
11912        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11913        assert!(
11914            listed.is_empty(),
11915            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11916        );
11917        let refused = handler
11918            .handle_control_frame(
11919                &client_ctx,
11920                route_open_frame(301, "aft", unique_project_root("listed-window")),
11921            )
11922            .await
11923            .unwrap();
11924        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11925        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11926        let duplicate = handler
11927            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11928            .await
11929            .unwrap();
11930        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11931
11932        hold.wait();
11933        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11934        let ack = module_rx.try_recv().unwrap().frame;
11935        assert_eq!(ack.header.ty, FrameType::HelloAck);
11936        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11937        assert_eq!(listed, vec![("aft".to_string(), true)]);
11938
11939        let (open_task, bind) = relay_route_open(
11940            &handler,
11941            client,
11942            &client_ctx.egress,
11943            &mut module_rx,
11944            303,
11945            "aft",
11946            "listed-after",
11947        )
11948        .await;
11949        assert!(handler
11950            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11951            .await
11952            .unwrap()
11953            .is_empty());
11954        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11955        published_route(&client_rx.try_recv().unwrap().frame);
11956    }
11957
11958    /// A catalog watcher keys on the generation: when it changes, it re-reads
11959    /// the listing. So the generation must advance exactly when the module
11960    /// appears. If it advanced while the module was still hidden, a watcher
11961    /// would read the new generation without the module, and the module would
11962    /// later appear under that same generation with nothing telling the
11963    /// watcher to look again.
11964    #[tokio::test]
11965    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11966        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11967        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11968        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11969
11970        hold.wait();
11971        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11972        assert!(listed.is_empty());
11973        assert_eq!(
11974            during, before,
11975            "the generation moved while the module was not listed"
11976        );
11977
11978        hold.wait();
11979        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11980        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11981        assert_eq!(listed, vec![("aft".to_string(), true)]);
11982        assert_eq!(
11983            after,
11984            before + 1,
11985            "the module appeared without exactly one generation bump"
11986        );
11987    }
11988
11989    /// A HELLO whose forwarding endpoint cannot be installed (here the
11990    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11991    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11992    /// and the catalog generation where it was.
11993    #[tokio::test]
11994    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11995        let registry = Arc::new(Registry::default());
11996        let handler = ControlHandler::with_forwarding(
11997            Arc::clone(&registry),
11998            Arc::new(ForwardingTable::default()),
11999        );
12000        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12001        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12002        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12003        drop(module_rx);
12004
12005        let replies = handler
12006            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12007            .await
12008            .unwrap();
12009        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12010        assert!(registry.get_module("aft").unwrap().is_none());
12011        assert!(registry
12012            .get_module_by_connection(module_ctx.connection_id)
12013            .unwrap()
12014            .is_none());
12015        assert_eq!(
12016            catalog_snapshot(&handler, &client_ctx, 301).await,
12017            (before, Vec::new())
12018        );
12019    }
12020
12021    /// A handler whose `route.open` parks after queueing the route.bind and
12022    /// before marking it enqueued, with module "aft" registered on connection
12023    /// 40. Returns the two-party barrier that parks and releases it.
12024    async fn route_open_held_after_bind_send() -> (
12025        ControlHandler,
12026        Arc<tokio::sync::Barrier>,
12027        RouteCtx,
12028        mpsc::Receiver<crate::router::OutboundFrame>,
12029    ) {
12030        let hold = Arc::new(tokio::sync::Barrier::new(2));
12031        let handler = ControlHandler::with_forwarding(
12032            Arc::new(Registry::default()),
12033            Arc::new(ForwardingTable::default()),
12034        )
12035        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12036        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12037        hello_via_sink(
12038            &handler,
12039            &module_ctx,
12040            &mut module_rx,
12041            hello_frame("aft", PROTOCOL_VERSION, 7),
12042        )
12043        .await;
12044        (handler, hold, module_ctx, module_rx)
12045    }
12046
12047    /// A fast module acks the route.bind, and the reader commits the route,
12048    /// before `route.open` has marked the relay enqueued. The daemon used to
12049    /// read the missing relay as an abandoned bind and send the module a
12050    /// GOODBYE for the route it had just committed: the client was told its
12051    /// route was open, the module unbound the channel, and the client's first
12052    /// request was dropped on the module side as unbound.
12053    #[tokio::test]
12054    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12055        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12056        let client = ConnectionId::new(41);
12057        let (client_ctx, mut client_rx) = route_ctx(client);
12058
12059        let (open_task, bind) = relay_route_open(
12060            &handler,
12061            client,
12062            &client_ctx.egress,
12063            &mut module_rx,
12064            200,
12065            "aft",
12066            "acked-before-mark",
12067        )
12068        .await;
12069        let (module_channel, module_epoch) = route_bind_channel(&bind);
12070        // route.open is now parked between queueing the bind and marking it.
12071        hold.wait().await;
12072
12073        // The module answers inside that window, and the route commits.
12074        assert!(handler
12075            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12076            .await
12077            .unwrap()
12078            .is_empty());
12079        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12080
12081        hold.wait().await;
12082        assert!(
12083            open_task.await.unwrap().is_empty(),
12084            "route.open answers accepted"
12085        );
12086
12087        // Every frame route.open could have sent is already queued, because it
12088        // has returned; the module must have been sent nothing.
12089        match module_rx.try_recv() {
12090            Err(mpsc::error::TryRecvError::Empty) => {}
12091            Ok(outbound) => panic!(
12092                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12093                outbound.frame.header.ty,
12094                outbound.frame.header.channel,
12095                outbound.frame.header.epoch
12096            ),
12097            Err(err) => panic!("module egress closed: {err}"),
12098        }
12099
12100        // The client's first request on the route is the next thing the module
12101        // reads, on the binding it just created.
12102        let router = Router::with_control_handler(Arc::new(handler.clone()));
12103        let request = Frame::build(
12104            FrameType::Request,
12105            Flags::new(true, Priority::Interactive, false),
12106            client_channel,
12107            client_epoch,
12108            201,
12109            b"first request".to_vec(),
12110        )
12111        .unwrap();
12112        router
12113            .route_for_connection(&client_ctx, request)
12114            .await
12115            .unwrap();
12116        let delivered = module_rx.try_recv().unwrap().frame;
12117        assert_eq!(delivered.header.ty, FrameType::Request);
12118        assert_eq!(
12119            (delivered.header.channel, delivered.header.epoch),
12120            (module_channel, module_epoch)
12121        );
12122        assert_eq!(delivered.body, b"first request");
12123    }
12124
12125    /// The client goes away while `route.open` is still queueing the bind, so
12126    /// the relay is dropped with no module reply. The teardown cannot send the
12127    /// GOODBYE itself (it could reach the module ahead of the bind), so
12128    /// `route.open` must send it once the bind is queued; otherwise the module
12129    /// keeps a binding the daemon will never commit.
12130    #[tokio::test]
12131    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12132        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12133        let client = ConnectionId::new(42);
12134        let (client_ctx, _client_rx) = route_ctx(client);
12135
12136        let (open_task, bind) = relay_route_open(
12137            &handler,
12138            client,
12139            &client_ctx.egress,
12140            &mut module_rx,
12141            300,
12142            "aft",
12143            "released-before-mark",
12144        )
12145        .await;
12146        let (module_channel, module_epoch) = route_bind_channel(&bind);
12147        hold.wait().await;
12148
12149        handler.cleanup_connection(client).unwrap();
12150        assert!(
12151            module_rx.try_recv().is_err(),
12152            "client teardown must not GOODBYE a bind route.open is still queueing"
12153        );
12154
12155        hold.wait().await;
12156        let response = open_task.await.unwrap();
12157        assert_eq!(response.len(), 1);
12158        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12159
12160        let goodbye = module_rx
12161            .try_recv()
12162            .expect("module is told to drop the binding the daemon abandoned")
12163            .frame;
12164        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12165        assert_eq!(
12166            (goodbye.header.channel, goodbye.header.epoch),
12167            (module_channel, module_epoch)
12168        );
12169        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12170    }
12171
12172    /// The module acks inside the same window, but the client's egress closed
12173    /// first, so the reader refuses to commit. The ack proves the module holds
12174    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
12175    /// send a second one.
12176    #[tokio::test]
12177    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12178        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12179        let client = ConnectionId::new(43);
12180        let (client_ctx, client_rx) = route_ctx(client);
12181
12182        let (open_task, bind) = relay_route_open(
12183            &handler,
12184            client,
12185            &client_ctx.egress,
12186            &mut module_rx,
12187            400,
12188            "aft",
12189            "gone-client-before-mark",
12190        )
12191        .await;
12192        let (module_channel, module_epoch) = route_bind_channel(&bind);
12193        hold.wait().await;
12194
12195        drop(client_rx);
12196        assert!(client_ctx.egress.is_closed());
12197        handler
12198            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12199            .await
12200            .unwrap();
12201
12202        hold.wait().await;
12203        let response = open_task.await.unwrap();
12204        assert_eq!(response.len(), 1);
12205        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12206
12207        let goodbye = module_rx
12208            .try_recv()
12209            .expect("module is told to drop the binding it acked")
12210            .frame;
12211        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12212        assert_eq!(
12213            (goodbye.header.channel, goodbye.header.epoch),
12214            (module_channel, module_epoch)
12215        );
12216        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12217        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12218    }
12219
12220    /// The fence at the module-loop boundary, stated as its own contract: which
12221    /// forwarding failures are allowed to end the module connection that is being
12222    /// served. A `ConnectionClosing` naming some client is about that client, and
12223    /// a module connection is shared; the same error naming the module's own
12224    /// connection is about this connection and must stay fatal, as must failures
12225    /// that are about the forwarding table itself.
12226    #[test]
12227    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12228        let handler = ControlHandler::default();
12229        let module_connection = ConnectionId::new(30);
12230        let client_connection = ConnectionId::new(31);
12231
12232        handler
12233            .refuse_to_end_module_connection_for_a_client(
12234                module_connection,
12235                77,
12236                ForwardingError::ConnectionClosing {
12237                    connection_id: client_connection,
12238                },
12239            )
12240            .expect("a closing client must never end the module connection");
12241
12242        assert!(matches!(
12243            handler.refuse_to_end_module_connection_for_a_client(
12244                module_connection,
12245                78,
12246                ForwardingError::ConnectionClosing {
12247                    connection_id: module_connection,
12248                },
12249            ),
12250            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12251                connection_id
12252            })) if connection_id == module_connection
12253        ));
12254        assert!(matches!(
12255            handler.refuse_to_end_module_connection_for_a_client(
12256                module_connection,
12257                79,
12258                ForwardingError::Poisoned,
12259            ),
12260            Err(RouterError::Forwarding(ForwardingError::Poisoned))
12261        ));
12262        assert!(matches!(
12263            handler.refuse_to_end_module_connection_for_a_client(
12264                module_connection,
12265                80,
12266                ForwardingError::StaleModuleEndpoint,
12267            ),
12268            Err(RouterError::Forwarding(
12269                ForwardingError::StaleModuleEndpoint
12270            ))
12271        ));
12272    }
12273
12274    /// The spawn-attestation guard is what stops a connected module from claiming
12275    /// another module's identity and being stamped `Reserved` for it. Every other
12276    /// test that supplies a consumer_identity supplies a CORRECT one, because a
12277    /// correct one is what the rest of the flow needs -- so the guard's rejection
12278    /// branch was never the subject of an assertion, only its acceptance branch.
12279    ///
12280    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
12281    /// whole subc-core library suite green; only the forwarding integration tests
12282    /// notice, and they notice for unrelated reasons. This test exists so the
12283    /// refusal itself is asserted where the guard lives: it fails if the identity
12284    /// check stops refusing, which is the direction that matters, since a guard
12285    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
12286    #[tokio::test]
12287    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12288        let registry = Arc::new(Registry::default());
12289        let forwarding = Arc::new(ForwardingTable::default());
12290        let supervisor = SupervisorHandle::new();
12291        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12292        let handler =
12293            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12294                .with_supervisor(supervisor);
12295
12296        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12297        hello_via_sink(
12298            &handler,
12299            &target_ctx,
12300            &mut target_rx,
12301            hello_frame("target", PROTOCOL_VERSION, 1),
12302        )
12303        .await;
12304
12305        // A real supervised module id presenting the wrong nonce. This is the
12306        // impersonation case: the attacker knows a privileged module_id, which is
12307        // public, and guesses at the nonce, which is not.
12308        let wrong_nonce = handler
12309            .handle_control_frame(
12310                &route_ctx(ConnectionId::new(91)).0,
12311                route_open_frame_with_admission_facts(
12312                    20,
12313                    "target",
12314                    unique_project_root("admission-facts"),
12315                    Some(subc_control::ConsumerIdentity {
12316                        module_id: "fed".to_string(),
12317                        launch_nonce: "not-the-real-nonce".to_string(),
12318                    }),
12319                    None,
12320                ),
12321            )
12322            .await
12323            .unwrap();
12324        assert_eq!(
12325            parse_error(&wrong_nonce[0])["code"],
12326            "bad_consumer_identity",
12327            "a mismatched launch nonce must be refused, not stamped Reserved"
12328        );
12329
12330        // A module id the supervisor never spawned at all, so no nonce exists to
12331        // compare against. An implementation that treats "no record" as "nothing
12332        // to check" fails open here while passing the case above.
12333        let never_spawned = handler
12334            .handle_control_frame(
12335                &route_ctx(ConnectionId::new(92)).0,
12336                route_open_frame_with_admission_facts(
12337                    21,
12338                    "target",
12339                    unique_project_root("admission-facts"),
12340                    Some(subc_control::ConsumerIdentity {
12341                        module_id: "never-spawned".to_string(),
12342                        launch_nonce: "any-nonce".to_string(),
12343                    }),
12344                    None,
12345                ),
12346            )
12347            .await
12348            .unwrap();
12349        assert_eq!(
12350            parse_error(&never_spawned[0])["code"],
12351            "bad_consumer_identity",
12352            "an unspawned module_id must be refused rather than accepted for lack of a record"
12353        );
12354    }
12355
12356    /// The refusal test above proves the guard says NO. Nothing proved it can say
12357    /// YES, and the difference is not academic: replacing the whole authorization
12358    /// with `false` -- admitting no consumer identity at all, revoking Reserved
12359    /// standing for every supervised module in the fleet -- leaves 110 of the 111
12360    /// library tests GREEN. The one that notices does so by HANGING, because it
12361    /// waits for a bind that can no longer happen.
12362    ///
12363    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
12364    /// or flaky test, invites a RETRY rather than an investigation, and the retry
12365    /// hangs too and gets blamed on the runner. So a total revocation of the
12366    /// daemon's trust grant would have shipped behind a symptom nobody attributes
12367    /// to code.
12368    ///
12369    /// The bias is structural rather than accidental. A REFUSAL looks like a
12370    /// failure someone writes a test for; a GRANT looks like the happy path. Every
12371    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
12372    /// suite for that reason, and this one is the purest case in the daemon.
12373    ///
12374    /// This test asserts the EFFECT rather than the absence of an error: the module
12375    /// receives a RouteBind and it carries `Reserved` naming the attested module.
12376    /// A guard that admitted nobody would produce no bind at all; one that admitted
12377    /// everybody would stamp the wrong principal, which the refusal test catches.
12378    #[tokio::test]
12379    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12380        let registry = Arc::new(Registry::default());
12381        let forwarding = Arc::new(ForwardingTable::default());
12382        let supervisor = SupervisorHandle::new();
12383        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12384        let handler =
12385            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12386                .with_supervisor(supervisor);
12387
12388        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12389        hello_via_sink(
12390            &handler,
12391            &target_ctx,
12392            &mut target_rx,
12393            hello_frame("target", PROTOCOL_VERSION, 1),
12394        )
12395        .await;
12396
12397        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12398        let route_handler = handler.clone();
12399        let route_task = tokio::spawn(async move {
12400            route_handler
12401                .handle_control_frame(
12402                    &client_ctx,
12403                    route_open_frame_with_admission_facts(
12404                        30,
12405                        "target",
12406                        unique_project_root("admission-facts"),
12407                        Some(subc_control::ConsumerIdentity {
12408                            module_id: "fed".to_string(),
12409                            launch_nonce: "fed-nonce".to_string(),
12410                        }),
12411                        None,
12412                    ),
12413                )
12414                .await
12415                .unwrap()
12416        });
12417
12418        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
12419        // the very mutation it exists to catch -- a guard that admits nobody -- no
12420        // bind is ever sent, so it HUNG rather than failing. That reproduces the
12421        // exact defect being fixed: a total revocation detected only as a stalled
12422        // suite, which reads as flakiness and invites a retry. An acceptance test
12423        // that waits for an effect must bound the wait, or a red becomes a hang.
12424        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12425            .await
12426            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12427            .expect("module control channel closed before route.bind");
12428        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12429        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12430            panic!("expected route.bind")
12431        };
12432        assert_eq!(
12433            principal,
12434            Some(Principal::Reserved {
12435                module_id: "fed".to_string()
12436            }),
12437            "a correctly attested consumer must be stamped Reserved for its own id"
12438        );
12439
12440        handler
12441            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12442            .await
12443            .unwrap();
12444        assert!(route_task.await.unwrap().is_empty());
12445        assert!(
12446            matches!(
12447                serde_json::from_slice::<ClientControlResponse>(
12448                    &client_rx.recv().await.unwrap().body
12449                )
12450                .unwrap(),
12451                ClientControlResponse::RouteOpen { .. }
12452            ),
12453            "the route must actually open, not merely avoid an error"
12454        );
12455    }
12456
12457    #[tokio::test(start_paused = true)]
12458    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12459        let registry = Arc::new(Registry::default());
12460        let forwarding = Arc::new(ForwardingTable::default());
12461        let supervisor = SupervisorHandle::new();
12462        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12463        let handler =
12464            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12465                .with_supervisor(supervisor);
12466
12467        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12468        hello_via_sink(
12469            &handler,
12470            &target_ctx,
12471            &mut target_rx,
12472            hello_frame("target", PROTOCOL_VERSION, 1),
12473        )
12474        .await;
12475
12476        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12477        let direct_handler = handler.clone();
12478        let direct_open = tokio::spawn(async move {
12479            direct_handler
12480                .handle_control_frame(
12481                    &direct_ctx,
12482                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
12483                )
12484                .await
12485                .unwrap()
12486        });
12487        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12488            .await
12489            .expect("no direct route.bind within 5s")
12490            .expect("target control channel closed before direct route.bind");
12491        handler
12492            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12493            .await
12494            .unwrap();
12495        assert!(direct_open.await.unwrap().is_empty());
12496        let _ = direct_rx.recv().await.unwrap();
12497
12498        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12499        let reserved_handler = handler.clone();
12500        let reserved_open = tokio::spawn(async move {
12501            reserved_handler
12502                .handle_control_frame(
12503                    &reserved_ctx,
12504                    route_open_frame_with_admission_facts(
12505                        3,
12506                        "target",
12507                        unique_project_root("admission-facts"),
12508                        Some(ConsumerIdentity {
12509                            module_id: "fed".to_string(),
12510                            launch_nonce: "fed-nonce".to_string(),
12511                        }),
12512                        None,
12513                    ),
12514                )
12515                .await
12516                .unwrap()
12517        });
12518        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12519            .await
12520            .expect("no reserved route.bind within 5s")
12521            .expect("target control channel closed before reserved route.bind");
12522        handler
12523            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12524            .await
12525            .unwrap();
12526        assert!(reserved_open.await.unwrap().is_empty());
12527        let _ = reserved_rx.recv().await.unwrap();
12528
12529        forwarding
12530            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12531            .unwrap();
12532        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12533        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12534            module_id: Some("target".to_string()),
12535        })
12536        .unwrap();
12537        let census_frame =
12538            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12539        let response = handler
12540            .handle_control_frame(&census_ctx, census_frame)
12541            .await
12542            .unwrap()
12543            .pop()
12544            .unwrap();
12545        let actual: Value = serde_json::from_slice(&response.body).unwrap();
12546        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12547        assert!(matches!(
12548            decoded,
12549            ClientControlResponse::SupervisorRoutes { .. }
12550        ));
12551        let routes = actual["modules"][0]["routes"].as_array().unwrap();
12552        assert_eq!(routes.len(), 2);
12553        assert!(routes.iter().all(|route| route["draining"] == true));
12554        // The census carries WHY: the reason the drain was begun with, in the
12555        // route.closing vocabulary, on every draining route this drain marked.
12556        assert!(
12557            routes.iter().all(|route| route["drain_reason"] == "reload"),
12558            "draining routes must name the drain's reason: {routes:?}"
12559        );
12560        assert!(routes.iter().any(|route| {
12561            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12562        }));
12563        assert!(routes.iter().any(|route| {
12564            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12565        }));
12566
12567        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12568            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12569        if std::env::var_os("UPDATE_GOLDEN").is_some() {
12570            std::fs::write(
12571                &golden_path,
12572                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12573            )
12574            .unwrap();
12575        }
12576        let expected: Value =
12577            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12578        assert_eq!(actual, expected);
12579    }
12580
12581    async fn query_live_roots(
12582        handler: &ControlHandler,
12583        module_ctx: &RouteCtx,
12584    ) -> ModuleControlResponseToModule {
12585        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12586        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12587        let response = handler
12588            .handle_control_frame(module_ctx, frame)
12589            .await
12590            .unwrap()
12591            .pop()
12592            .unwrap();
12593        serde_json::from_slice(&response.body).unwrap()
12594    }
12595
12596    #[tokio::test(start_paused = true)]
12597    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12598        let registry = Arc::new(Registry::default());
12599        let forwarding = Arc::new(ForwardingTable::default());
12600        let handler = ControlHandler::with_forwarding(registry, forwarding);
12601        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12602        hello_via_sink(
12603            &handler,
12604            &target_ctx,
12605            &mut target_rx,
12606            hello_frame("target", PROTOCOL_VERSION, 1),
12607        )
12608        .await;
12609        let root = unique_project_root("live-roots-known");
12610        let path = ProjectRootId::from_path_allowing_missing(root.path())
12611            .unwrap()
12612            .as_path()
12613            .to_path_buf();
12614        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12615        let open_handler = handler.clone();
12616        let opened = tokio::spawn(async move {
12617            open_handler
12618                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12619                .await
12620                .unwrap()
12621        });
12622        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12623            .await
12624            .unwrap()
12625            .unwrap();
12626        handler
12627            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12628            .await
12629            .unwrap();
12630        assert!(opened.await.unwrap().is_empty());
12631        let _ = client_rx.recv().await.unwrap();
12632
12633        let root = unique_project_root("live-roots-pending");
12634        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12635            .unwrap()
12636            .as_path()
12637            .to_path_buf();
12638        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12639        let open_handler = handler.clone();
12640        let pending = tokio::spawn(async move {
12641            open_handler
12642                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12643                .await
12644                .unwrap()
12645        });
12646        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12647            .await
12648            .unwrap()
12649            .unwrap();
12650        let actual = query_live_roots(&handler, &target_ctx).await;
12651        let ModuleControlResponseToModule::LiveRoots {
12652            roots,
12653            unknown_root_bindings,
12654            total_bindings,
12655        } = actual
12656        else {
12657            panic!("expected live roots")
12658        };
12659        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12660        assert_eq!(unknown_root_bindings, 0);
12661        assert_eq!(
12662            roots.len(),
12663            2,
12664            "root-known arm must retain each canonical root"
12665        );
12666        assert_eq!(
12667            total_bindings,
12668            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12669        );
12670        let counts = roots
12671            .iter()
12672            .map(|root| (root.project_root.clone(), root.bound, root.pending))
12673            .collect::<Vec<_>>();
12674        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12675        expected.sort_by(|a, b| a.0.cmp(&b.0));
12676        assert_eq!(
12677            counts, expected,
12678            "roots must sort by path and count pending separately"
12679        );
12680        handler
12681            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12682            .await
12683            .unwrap();
12684        assert!(pending.await.unwrap().is_empty());
12685    }
12686
12687    #[tokio::test(start_paused = true)]
12688    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12689        let forwarding = Arc::new(ForwardingTable::default());
12690        let handler =
12691            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12692        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12693        hello_via_sink(
12694            &handler,
12695            &target_ctx,
12696            &mut target_rx,
12697            hello_frame("target", PROTOCOL_VERSION, 1),
12698        )
12699        .await;
12700        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12701        let pending = forwarding
12702            .begin_route_bind_relay_for_test(
12703                client_ctx.connection_id,
12704                client_ctx.egress.clone(),
12705                2,
12706                "target",
12707            )
12708            .unwrap();
12709        forwarding
12710            .complete_pending_relay(
12711                target_ctx.connection_id,
12712                pending.corr,
12713                RouteBindRelayOutcome::Accepted,
12714            )
12715            .unwrap();
12716        let actual = query_live_roots(&handler, &target_ctx).await;
12717        let ModuleControlResponseToModule::LiveRoots {
12718            roots,
12719            unknown_root_bindings,
12720            total_bindings,
12721        } = actual
12722        else {
12723            panic!("expected live roots")
12724        };
12725        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12726        assert_eq!(
12727            unknown_root_bindings, 1,
12728            "unknown-root arm must not read as no bindings"
12729        );
12730        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12731        assert_eq!(
12732            total_bindings,
12733            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12734        );
12735    }
12736
12737    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12738    /// so the ack has to be on its outbound queue before the module is
12739    /// routable. The connection loop writes a handler's replies only after the
12740    /// handler returns; this test stops in exactly that gap, runs a real
12741    /// route.open from another connection, and only then writes whatever the
12742    /// HELLO handler returned, the way the loop would. If the ack were still a
12743    /// reply, the route.bind request would reach the module first.
12744    #[tokio::test(start_paused = true)]
12745    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12746        let forwarding = Arc::new(ForwardingTable::default());
12747        let handler =
12748            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12749        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12750        let replies = handler
12751            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12752            .await
12753            .unwrap();
12754        let queued_by_hello = module_rx.len();
12755
12756        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12757        let open_handler = handler.clone();
12758        let open = tokio::spawn(async move {
12759            open_handler
12760                .handle_control_frame(
12761                    &client_ctx,
12762                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12763                )
12764                .await
12765                .unwrap()
12766        });
12767        // Let the route.open run until its route.bind is on the module's queue.
12768        let mut spins = 0;
12769        while module_rx.len() == queued_by_hello {
12770            spins += 1;
12771            assert!(spins < 10_000, "route.open never queued a route.bind");
12772            tokio::task::yield_now().await;
12773        }
12774
12775        // Now the connection loop's half: write the HELLO handler's replies.
12776        for reply in replies {
12777            module_ctx.egress.send(reply).await.unwrap();
12778        }
12779
12780        let first = module_rx.recv().await.unwrap().frame;
12781        assert_eq!(
12782            first.header.ty,
12783            FrameType::HelloAck,
12784            "the first frame a registering module reads must be its HELLO_ACK"
12785        );
12786        assert_eq!(first.header.corr, 7);
12787        let second = module_rx.recv().await.unwrap().frame;
12788        assert_eq!(second.header.ty, FrameType::Request);
12789        assert!(
12790            matches!(
12791                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12792                ModuleControlRequest::RouteBind { .. }
12793            ),
12794            "the route.bind follows the ack"
12795        );
12796        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12797
12798        handler
12799            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12800            .await
12801            .unwrap();
12802        assert!(open.await.unwrap().is_empty());
12803        let _ = client_rx.recv().await.unwrap();
12804    }
12805
12806    #[tokio::test(start_paused = true)]
12807    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12808        let handler = ControlHandler::with_forwarding(
12809            Arc::new(Registry::default()),
12810            Arc::new(ForwardingTable::default()),
12811        );
12812        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12813        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12814        hello_via_sink(
12815            &handler,
12816            &first_ctx,
12817            &mut first_rx,
12818            hello_frame("first", PROTOCOL_VERSION, 1),
12819        )
12820        .await;
12821        hello_via_sink(
12822            &handler,
12823            &second_ctx,
12824            &mut second_rx,
12825            hello_frame("second", PROTOCOL_VERSION, 2),
12826        )
12827        .await;
12828        let root = unique_project_root("second-only");
12829        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12830        let cloned = handler.clone();
12831        let open = tokio::spawn(async move {
12832            cloned
12833                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12834                .await
12835                .unwrap()
12836        });
12837        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12838            .await
12839            .unwrap()
12840            .unwrap();
12841        let first = query_live_roots(&handler, &first_ctx).await;
12842        let second = query_live_roots(&handler, &second_ctx).await;
12843        assert!(
12844            matches!(
12845                first,
12846                ModuleControlResponseToModule::LiveRoots {
12847                    total_bindings: 0,
12848                    ..
12849                }
12850            ),
12851            "cross-module scope must not expose another module's roots"
12852        );
12853        assert!(
12854            matches!(
12855                second,
12856                ModuleControlResponseToModule::LiveRoots {
12857                    total_bindings: 1,
12858                    ..
12859                }
12860            ),
12861            "second module must see its pending route"
12862        );
12863        handler
12864            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12865            .await
12866            .unwrap();
12867        assert!(open.await.unwrap().is_empty());
12868    }
12869
12870    #[tokio::test(start_paused = true)]
12871    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12872        let handler = ControlHandler::with_forwarding(
12873            Arc::new(Registry::default()),
12874            Arc::new(ForwardingTable::default()),
12875        );
12876        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12877        hello_via_sink(
12878            &handler,
12879            &target_ctx,
12880            &mut target_rx,
12881            hello_frame("target", PROTOCOL_VERSION, 1),
12882        )
12883        .await;
12884        let actual = query_live_roots(&handler, &target_ctx).await;
12885        let ModuleControlResponseToModule::LiveRoots {
12886            roots,
12887            unknown_root_bindings,
12888            total_bindings,
12889        } = actual
12890        else {
12891            panic!("expected live roots")
12892        };
12893        assert!(roots.is_empty());
12894        assert_eq!(unknown_root_bindings, 0);
12895        assert_eq!(total_bindings, 0);
12896        assert_eq!(
12897            total_bindings,
12898            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12899        );
12900    }
12901
12902    /// Read the vendored fed corpus rather than hand-building a package.
12903    ///
12904    /// A hand-built object encodes what the test author believed the carrier
12905    /// emits. These vectors are what it actually emits, and one of them exists
12906    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
12907    /// ignores additive unknown fields at the traversal emit terminus."
12908    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12909        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12910            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12911        let text = std::fs::read_to_string(&path)
12912            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12913        let vectors: Vec<(String, Value)> = text
12914            .lines()
12915            .filter(|line| !line.trim().is_empty())
12916            .map(|line| {
12917                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12918                let id = entry["corpus_id"]
12919                    .as_str()
12920                    .expect("every vector carries a corpus_id")
12921                    .to_string();
12922                (id, entry["package"].clone())
12923            })
12924            .collect();
12925        // Pin the count: a corpus that silently shrinks would take its coverage
12926        // with it, and a suite reading N-1 vectors reports the same clean pass
12927        // as one reading N.
12928        assert_eq!(
12929            vectors.len(),
12930            3,
12931            "vendored fed corpus changed size; re-sync from subc-federation"
12932        );
12933
12934        // Pin what makes the corpus DISCRIMINATING, not just present.
12935        //
12936        // The relay test below takes its expected value from the corpus, so the
12937        // corpus supplies the test's power to detect a lossy relay rather than
12938        // its correctness. A relay that dropped unrecognised fields would still
12939        // be caught -- but only by a package carrying fields it does not know.
12940        // Shrink every package to the handful of keys any implementation would
12941        // recognise and the test keeps passing over an input that can no longer
12942        // fail, which is the same clean green as a corpus that shrank away.
12943        //
12944        // So assert the precondition rather than duplicating the packages here:
12945        // at least one vector must carry a field beyond the small common set.
12946        // That is one claim to maintain instead of nine, and it fails loudly if
12947        // a re-sync ever flattens the corpus.
12948        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12949        let richest = vectors
12950            .iter()
12951            .filter_map(|(_, package)| package.as_object())
12952            .map(|object| {
12953                object
12954                    .keys()
12955                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12956                    .count()
12957            })
12958            .max()
12959            .unwrap_or(0);
12960        assert!(
12961            richest >= 2,
12962            "vendored corpus no longer carries a package with unmodelled fields, \
12963             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12964        );
12965
12966        vectors
12967    }
12968
12969    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12970    ///
12971    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12972    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12973    /// three-key object, so a relay that quietly dropped fields it did not
12974    /// recognise would satisfy it. These vectors carry nine keys including ones
12975    /// this crate has no type for, so a typed relay fails here and only here.
12976    #[tokio::test]
12977    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12978        for (corpus_id, package) in fed_admission_facts_vectors() {
12979            let registry = Arc::new(Registry::default());
12980            let forwarding = Arc::new(ForwardingTable::default());
12981            let supervisor = SupervisorHandle::new();
12982            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12983            let handler =
12984                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12985                    .with_supervisor(supervisor)
12986                    .with_admission_facts_config(
12987                        Some("fed".to_string()),
12988                        Some(vec!["target".to_string()]),
12989                    );
12990
12991            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12992            hello_via_sink(
12993                &handler,
12994                &target_ctx,
12995                &mut target_rx,
12996                hello_frame("target", PROTOCOL_VERSION, 1),
12997            )
12998            .await;
12999
13000            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
13001            let route_handler = handler.clone();
13002            let expected = package.clone();
13003            let route_task = tokio::spawn(async move {
13004                route_handler
13005                    .handle_control_frame(
13006                        &client_ctx,
13007                        route_open_frame_with_admission_facts(
13008                            20,
13009                            "target",
13010                            unique_project_root("admission-facts"),
13011                            Some(subc_control::ConsumerIdentity {
13012                                module_id: "fed".to_string(),
13013                                launch_nonce: "fed-nonce".to_string(),
13014                            }),
13015                            Some(package),
13016                        ),
13017                    )
13018                    .await
13019                    .unwrap()
13020            });
13021
13022            let bind_frame = target_rx.recv().await.unwrap();
13023            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13024            let ModuleControlRequest::RouteBind {
13025                admission_facts, ..
13026            } = bind
13027            else {
13028                panic!("{corpus_id}: expected route.bind")
13029            };
13030            assert_eq!(
13031                admission_facts,
13032                Some(expected),
13033                "{corpus_id}: relay must not add, drop or reshape any field"
13034            );
13035
13036            handler
13037                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13038                .await
13039                .unwrap();
13040            route_task.await.unwrap();
13041        }
13042    }
13043
13044    #[tokio::test]
13045    async fn route_bind_never_relays_the_openers_build_class() {
13046        let registry = Arc::new(Registry::default());
13047        let forwarding = Arc::new(ForwardingTable::default());
13048        let handler =
13049            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13050        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13051        hello_via_sink(
13052            &handler,
13053            &target_ctx,
13054            &mut target_rx,
13055            hello_frame("target", PROTOCOL_VERSION, 1),
13056        )
13057        .await;
13058
13059        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13060        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13061        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13062            busy_refusals: None,
13063            target: RouteTarget::ToolProvider {
13064                module_id: "target".to_string(),
13065            },
13066            identity,
13067            consumer_identity: None,
13068            consumer_capabilities: None,
13069            role_versions: None,
13070            admission_facts: None,
13071            scope: None,
13072        })
13073        .unwrap();
13074        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13075
13076        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13077        let route_handler = handler.clone();
13078        let route_task = tokio::spawn(async move {
13079            route_handler
13080                .handle_control_frame(&client_ctx, open)
13081                .await
13082                .unwrap()
13083        });
13084        let bind_frame = target_rx.recv().await.unwrap();
13085        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13086        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13087            panic!("expected route.bind")
13088        };
13089        assert_eq!(
13090            identity.build_class, None,
13091            "route.bind carried the build class"
13092        );
13093        assert!(
13094            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13095            "the route.bind bytes still name build_class"
13096        );
13097        handler
13098            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13099            .await
13100            .unwrap();
13101        route_task.await.unwrap();
13102    }
13103
13104    #[tokio::test]
13105    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13106        let registry = Arc::new(Registry::default());
13107        let forwarding = Arc::new(ForwardingTable::default());
13108        let supervisor = SupervisorHandle::new();
13109        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13110        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13111        let handler =
13112            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13113                .with_supervisor(supervisor)
13114                .with_admission_facts_config(
13115                    Some("fed".to_string()),
13116                    Some(vec!["target".to_string()]),
13117                );
13118
13119        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13120        hello_via_sink(
13121            &handler,
13122            &target_ctx,
13123            &mut target_rx,
13124            hello_frame("target", PROTOCOL_VERSION, 1),
13125        )
13126        .await;
13127        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13128        hello_via_sink(
13129            &handler,
13130            &other_ctx,
13131            &mut other_rx,
13132            hello_frame("other", PROTOCOL_VERSION, 2),
13133        )
13134        .await;
13135
13136        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13137        let expected_facts = facts.clone();
13138        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13139        let route_handler = handler.clone();
13140        let route_task = tokio::spawn(async move {
13141            route_handler
13142                .handle_control_frame(
13143                    &client_ctx,
13144                    route_open_frame_with_admission_facts(
13145                        10,
13146                        "target",
13147                        unique_project_root("admission-facts"),
13148                        Some(subc_control::ConsumerIdentity {
13149                            module_id: "fed".to_string(),
13150                            launch_nonce: "fed-nonce".to_string(),
13151                        }),
13152                        Some(facts.clone()),
13153                    ),
13154                )
13155                .await
13156                .unwrap()
13157        });
13158        let bind_frame = target_rx.recv().await.unwrap();
13159        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13160        let ModuleControlRequest::RouteBind {
13161            admission_facts, ..
13162        } = bind
13163        else {
13164            panic!("expected route.bind")
13165        };
13166        assert_eq!(admission_facts, Some(expected_facts));
13167        handler
13168            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13169            .await
13170            .unwrap();
13171        assert!(route_task.await.unwrap().is_empty());
13172        assert!(matches!(
13173            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13174                .unwrap(),
13175            ClientControlResponse::RouteOpen { .. }
13176        ));
13177
13178        let direct = handler
13179            .handle_control_frame(
13180                &route_ctx(ConnectionId::new(73)).0,
13181                route_open_frame_with_admission_facts(
13182                    11,
13183                    "target",
13184                    unique_project_root("admission-facts"),
13185                    None,
13186                    Some(json!({"x": 1})),
13187                ),
13188            )
13189            .await
13190            .unwrap();
13191        assert_eq!(
13192            parse_error(&direct[0])["code"],
13193            "admission_facts_not_permitted"
13194        );
13195
13196        let different_reserved = handler
13197            .handle_control_frame(
13198                &route_ctx(ConnectionId::new(77)).0,
13199                route_open_frame_with_admission_facts(
13200                    15,
13201                    "target",
13202                    unique_project_root("admission-facts"),
13203                    Some(subc_control::ConsumerIdentity {
13204                        module_id: "other".to_string(),
13205                        launch_nonce: "other-nonce".to_string(),
13206                    }),
13207                    Some(json!({"x": 1})),
13208                ),
13209            )
13210            .await
13211            .unwrap();
13212        assert_eq!(
13213            parse_error(&different_reserved[0])["code"],
13214            "admission_facts_not_permitted"
13215        );
13216
13217        let other_target = handler
13218            .handle_control_frame(
13219                &route_ctx(ConnectionId::new(74)).0,
13220                route_open_frame_with_admission_facts(
13221                    12,
13222                    "other",
13223                    unique_project_root("admission-facts"),
13224                    Some(subc_control::ConsumerIdentity {
13225                        module_id: "fed".to_string(),
13226                        launch_nonce: "fed-nonce".to_string(),
13227                    }),
13228                    Some(json!({"x": 1})),
13229                ),
13230            )
13231            .await
13232            .unwrap();
13233        assert_eq!(
13234            parse_error(&other_target[0])["code"],
13235            "admission_facts_target_not_allowed"
13236        );
13237
13238        let nonexistent = handler
13239            .handle_control_frame(
13240                &route_ctx(ConnectionId::new(75)).0,
13241                route_open_frame_with_admission_facts(
13242                    13,
13243                    "missing",
13244                    unique_project_root("admission-facts"),
13245                    None,
13246                    Some(json!({"x": 1})),
13247                ),
13248            )
13249            .await
13250            .unwrap();
13251        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13252
13253        let described = handler
13254            .handle_control_frame(
13255                &route_ctx(ConnectionId::new(76)).0,
13256                Frame::build(
13257                    FrameType::Request,
13258                    control_flags(),
13259                    0,
13260                    0,
13261                    14,
13262                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13263                )
13264                .unwrap(),
13265            )
13266            .await
13267            .unwrap();
13268        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13269            serde_json::from_slice(&described[0].body).unwrap()
13270        else {
13271            panic!("expected server.describe response")
13272        };
13273        assert!(capabilities
13274            .iter()
13275            .any(|cap| cap == "admission_facts_relay_v1"));
13276    }
13277
13278    #[tokio::test]
13279    async fn admission_facts_without_configured_carrier_are_rejected() {
13280        let registry = Arc::new(Registry::default());
13281        let forwarding = Arc::new(ForwardingTable::default());
13282        let handler = ControlHandler::with_forwarding(registry, forwarding);
13283        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13284        hello_via_sink(
13285            &handler,
13286            &target_ctx,
13287            &mut target_rx,
13288            hello_frame("target", PROTOCOL_VERSION, 1),
13289        )
13290        .await;
13291
13292        let responses = handler
13293            .handle_control_frame(
13294                &route_ctx(ConnectionId::new(79)).0,
13295                route_open_frame_with_admission_facts(
13296                    16,
13297                    "target",
13298                    unique_project_root("admission-facts"),
13299                    None,
13300                    Some(json!({"x": 1})),
13301                ),
13302            )
13303            .await
13304            .unwrap();
13305        assert_eq!(
13306            parse_error(&responses[0])["code"],
13307            "admission_facts_not_permitted"
13308        );
13309    }
13310
13311    #[tokio::test]
13312    async fn route_open_relays_consumer_capabilities_verbatim() {
13313        let registry = Arc::new(Registry::default());
13314        let forwarding = Arc::new(ForwardingTable::default());
13315        let handler =
13316            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13317        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13318        hello_via_sink(
13319            &handler,
13320            &module_ctx,
13321            &mut module_rx,
13322            hello_frame("aft", PROTOCOL_VERSION, 7),
13323        )
13324        .await;
13325
13326        let expected = vec!["elicitation".to_string(), "roots".to_string()];
13327        let expected_for_request = expected.clone();
13328        let project_root = unique_project_root("consumer-capabilities-present");
13329        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13330        let route_handler = handler.clone();
13331        let route_task = tokio::spawn(async move {
13332            route_handler
13333                .handle_control_frame(
13334                    &client_ctx,
13335                    route_open_frame_with_consumer_capabilities(
13336                        401,
13337                        "aft",
13338                        project_root,
13339                        Some(expected_for_request),
13340                    ),
13341                )
13342                .await
13343                .unwrap()
13344        });
13345        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13346            .await
13347            .unwrap()
13348            .unwrap();
13349        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13350        let ModuleControlRequest::RouteBind {
13351            consumer_capabilities,
13352            ..
13353        } = bind
13354        else {
13355            panic!("expected route.bind request, got {bind:?}");
13356        };
13357        assert_eq!(consumer_capabilities, Some(expected.clone()));
13358
13359        handler
13360            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13361            .await
13362            .unwrap();
13363        let route_response = route_task.await.unwrap();
13364        assert!(route_response.is_empty());
13365        let published = client_rx.recv().await.unwrap();
13366        assert!(matches!(
13367            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13368            ClientControlResponse::RouteOpen { .. }
13369        ));
13370    }
13371
13372    #[tokio::test]
13373    async fn route_open_without_consumer_capabilities_relays_none() {
13374        let registry = Arc::new(Registry::default());
13375        let forwarding = Arc::new(ForwardingTable::default());
13376        let handler =
13377            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13378        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13379        hello_via_sink(
13380            &handler,
13381            &module_ctx,
13382            &mut module_rx,
13383            hello_frame("aft", PROTOCOL_VERSION, 7),
13384        )
13385        .await;
13386
13387        let project_root = unique_project_root("consumer-capabilities-absent");
13388        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13389        let route_handler = handler.clone();
13390        let route_task = tokio::spawn(async move {
13391            route_handler
13392                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13393                .await
13394                .unwrap()
13395        });
13396        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13397            .await
13398            .unwrap()
13399            .unwrap();
13400        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13401        let ModuleControlRequest::RouteBind {
13402            consumer_capabilities,
13403            ..
13404        } = bind
13405        else {
13406            panic!("expected route.bind request, got {bind:?}");
13407        };
13408        assert_eq!(consumer_capabilities, None);
13409
13410        handler
13411            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13412            .await
13413            .unwrap();
13414        let route_response = route_task.await.unwrap();
13415        assert!(route_response.is_empty());
13416        let published = client_rx.recv().await.unwrap();
13417        assert!(matches!(
13418            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13419            ClientControlResponse::RouteOpen { .. }
13420        ));
13421    }
13422
13423    /// Opens a route to a freshly registered `aft` with `sent` as the
13424    /// route.open's role_versions, acks the bind, and returns the role_versions
13425    /// the module's bind carried.
13426    async fn bind_role_versions_for(
13427        sent: Option<BTreeMap<String, String>>,
13428        connection: u64,
13429    ) -> Option<BTreeMap<String, String>> {
13430        let registry = Arc::new(Registry::default());
13431        let forwarding = Arc::new(ForwardingTable::default());
13432        let handler =
13433            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13434        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13435        hello_via_sink(
13436            &handler,
13437            &module_ctx,
13438            &mut module_rx,
13439            hello_frame("aft", PROTOCOL_VERSION, 7),
13440        )
13441        .await;
13442        let project_root = unique_project_root("role-versions");
13443        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13444        let route_handler = handler.clone();
13445        let route_task = tokio::spawn(async move {
13446            route_handler
13447                .handle_control_frame(
13448                    &client_ctx,
13449                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
13450                )
13451                .await
13452                .unwrap()
13453        });
13454        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13455            .await
13456            .expect("a well-formed route.open reaches the module as a bind")
13457            .unwrap();
13458        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13459        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13460            panic!("expected route.bind request, got {bind:?}");
13461        };
13462        handler
13463            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13464            .await
13465            .unwrap();
13466        assert!(route_task.await.unwrap().is_empty());
13467        let published = client_rx.recv().await.unwrap();
13468        assert!(matches!(
13469            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13470            ClientControlResponse::RouteOpen { .. }
13471        ));
13472        role_versions
13473    }
13474
13475    #[tokio::test]
13476    async fn route_open_relays_role_versions_verbatim() {
13477        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13478        assert_eq!(
13479            bind_role_versions_for(Some(sent.clone()), 141).await,
13480            Some(sent)
13481        );
13482    }
13483
13484    /// An empty map declares nothing, so the provider sees no field rather
13485    /// than an empty object it would have to treat as a second "none".
13486    #[tokio::test]
13487    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13488        assert_eq!(bind_role_versions_for(None, 143).await, None);
13489        assert_eq!(
13490            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13491            None
13492        );
13493    }
13494
13495    #[tokio::test]
13496    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13497        let registry = Arc::new(Registry::default());
13498        let forwarding = Arc::new(ForwardingTable::default());
13499        let handler =
13500            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13501        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13502        hello_via_sink(
13503            &handler,
13504            &module_ctx,
13505            &mut module_rx,
13506            hello_frame("aft", PROTOCOL_VERSION, 7),
13507        )
13508        .await;
13509
13510        let nine: BTreeMap<String, String> = (0..9)
13511            .map(|index| (format!("role-{index}"), "v1".to_string()))
13512            .collect();
13513        for (label, malformed) in [
13514            (
13515                "invalid role name",
13516                role_versions(&[("Tool_Provider", "v1")]),
13517            ),
13518            ("invalid version", role_versions(&[("tool-provider", "v0")])),
13519            ("nine entries", nine),
13520        ] {
13521            // A refused open answers at once; one that reached the module
13522            // would wait for its bind ack and trip this timeout.
13523            let responses = tokio::time::timeout(
13524                Duration::from_secs(1),
13525                handler.handle_control_frame(
13526                    &route_ctx(ConnectionId::new(148)).0,
13527                    route_open_frame_with_role_versions(
13528                        404,
13529                        "aft",
13530                        unique_project_root("role-versions-malformed"),
13531                        Some(malformed),
13532                    ),
13533                ),
13534            )
13535            .await
13536            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13537            .unwrap();
13538            assert_eq!(responses.len(), 1, "{label}");
13539            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13540            let error = parse_error(&responses[0]);
13541            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13542            assert_eq!(
13543                error["detail"]["field"], "role_versions",
13544                "{label}: {error}"
13545            );
13546            assert!(
13547                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13548                "{label}: a malformed declaration is terminal"
13549            );
13550            assert!(
13551                module_rx.try_recv().is_err(),
13552                "{label}: the module must never see a bind"
13553            );
13554        }
13555    }
13556
13557    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
13558    /// consumer can tell this daemon forwards the field from one that would
13559    /// drop it.
13560    #[tokio::test]
13561    async fn route_role_versions_capability_is_advertised() {
13562        let handler = ControlHandler::new(Arc::new(Registry::default()));
13563        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13564        let ack = hello_via_sink(
13565            &handler,
13566            &ctx,
13567            &mut rx,
13568            hello_frame("m", PROTOCOL_VERSION, 1),
13569        )
13570        .await;
13571        let ack = parse_ack(&ack);
13572        assert!(
13573            ack.subc_capabilities
13574                .iter()
13575                .any(|c| c == "route-role-versions/v1"),
13576            "{:?}",
13577            ack.subc_capabilities
13578        );
13579
13580        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13581        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13582        let reply = handler
13583            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13584            .await
13585            .unwrap()
13586            .pop()
13587            .unwrap();
13588        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13589            serde_json::from_slice(&reply.body).unwrap()
13590        else {
13591            panic!("not a server.describe reply");
13592        };
13593        assert!(
13594            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13595            "{capabilities:?}"
13596        );
13597    }
13598
13599    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
13600    /// catalog entry without `events` (the module declares none) from a daemon
13601    /// that never reports declared events.
13602    #[tokio::test]
13603    async fn catalog_events_capability_is_advertised() {
13604        let handler = ControlHandler::new(Arc::new(Registry::default()));
13605        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13606        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13607        let reply = handler
13608            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13609            .await
13610            .unwrap()
13611            .pop()
13612            .unwrap();
13613        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13614            serde_json::from_slice(&reply.body).unwrap()
13615        else {
13616            panic!("not a server.describe reply");
13617        };
13618        assert!(
13619            capabilities
13620                .iter()
13621                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13622            "{capabilities:?}"
13623        );
13624    }
13625
13626    #[tokio::test]
13627    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13628        let handler = ControlHandler::new(Arc::new(Registry::default()));
13629        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13630        let ack = hello_via_sink(
13631            &handler,
13632            &module,
13633            &mut module_rx,
13634            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13635        )
13636        .await;
13637        let ack = parse_ack(&ack);
13638        assert!(
13639            ack.subc_capabilities
13640                .iter()
13641                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13642            "{:?}",
13643            ack.subc_capabilities
13644        );
13645        assert!(ack
13646            .subc_capabilities
13647            .iter()
13648            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13649
13650        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13651        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13652        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13653        let reply = handler
13654            .handle_control_frame(&client, frame)
13655            .await
13656            .unwrap()
13657            .pop()
13658            .unwrap();
13659        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13660            serde_json::from_slice(&reply.body).unwrap()
13661        else {
13662            panic!("not a server.describe reply");
13663        };
13664        assert!(
13665            capabilities
13666                .iter()
13667                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13668            "{capabilities:?}"
13669        );
13670        assert!(capabilities
13671            .iter()
13672            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13673    }
13674
13675    #[tokio::test]
13676    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13677        let handler = ControlHandler::new(Arc::new(Registry::default()));
13678        let (module, mut rx) = route_ctx(ConnectionId::new(156));
13679        let ack = hello_via_sink(
13680            &handler,
13681            &module,
13682            &mut rx,
13683            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13684        )
13685        .await;
13686        assert!(parse_ack(&ack)
13687            .subc_capabilities
13688            .iter()
13689            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13690        assert!(parse_ack(&ack)
13691            .subc_capabilities
13692            .iter()
13693            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13694        let (client, _rx) = route_ctx(ConnectionId::new(157));
13695        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13696        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13697        let reply = handler
13698            .handle_control_frame(&client, frame)
13699            .await
13700            .unwrap()
13701            .pop()
13702            .unwrap();
13703        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13704            serde_json::from_slice(&reply.body).unwrap()
13705        else {
13706            panic!("server.describe reply")
13707        };
13708        assert!(capabilities
13709            .iter()
13710            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13711        assert!(capabilities
13712            .iter()
13713            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13714    }
13715
13716    #[tokio::test]
13717    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13718        let handler = ControlHandler::new(Arc::new(Registry::default()));
13719        let (module, mut rx) = route_ctx(ConnectionId::new(158));
13720        let ack = hello_via_sink(
13721            &handler,
13722            &module,
13723            &mut rx,
13724            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13725        )
13726        .await;
13727        assert!(parse_ack(&ack)
13728            .subc_capabilities
13729            .iter()
13730            .any(|cap| cap == "scope-workdir/v1"));
13731        let (client, _rx) = route_ctx(ConnectionId::new(159));
13732        let frame = Frame::build(
13733            FrameType::Request,
13734            control_flags(),
13735            0,
13736            0,
13737            5,
13738            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13739        )
13740        .unwrap();
13741        let reply = handler
13742            .handle_control_frame(&client, frame)
13743            .await
13744            .unwrap()
13745            .remove(0);
13746        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13747            serde_json::from_slice(&reply.body).unwrap()
13748        else {
13749            panic!("server.describe reply")
13750        };
13751        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13752    }
13753
13754    #[tokio::test]
13755    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13756        let handler = ControlHandler::new(Arc::new(Registry::default()));
13757        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13758        let ack = hello_via_sink(
13759            &handler,
13760            &module,
13761            &mut rx,
13762            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13763        )
13764        .await;
13765        assert!(parse_ack(&ack)
13766            .subc_capabilities
13767            .iter()
13768            .any(|cap| cap == "worker-runs/v1"));
13769        let (client, _rx) = route_ctx(ConnectionId::new(161));
13770        let frame = Frame::build(
13771            FrameType::Request,
13772            control_flags(),
13773            0,
13774            0,
13775            5,
13776            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13777        )
13778        .unwrap();
13779        let reply = handler
13780            .handle_control_frame(&client, frame)
13781            .await
13782            .unwrap()
13783            .remove(0);
13784        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13785            serde_json::from_slice(&reply.body).unwrap()
13786        else {
13787            panic!("server.describe reply")
13788        };
13789        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13790    }
13791
13792    #[tokio::test]
13793    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13794        let handler = ControlHandler::new(Arc::new(Registry::default()));
13795        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13796        let ack = hello_via_sink(
13797            &handler,
13798            &module,
13799            &mut rx,
13800            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13801        )
13802        .await;
13803        assert!(parse_ack(&ack)
13804            .subc_capabilities
13805            .iter()
13806            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13807        let (client, _rx) = route_ctx(ConnectionId::new(155));
13808        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13809        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13810        let reply = handler
13811            .handle_control_frame(&client, frame)
13812            .await
13813            .unwrap()
13814            .pop()
13815            .unwrap();
13816        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13817            serde_json::from_slice(&reply.body).unwrap()
13818        else {
13819            panic!("not a server.describe reply")
13820        };
13821        assert!(capabilities
13822            .iter()
13823            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13824    }
13825
13826    #[tokio::test]
13827    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13828        let registry = Arc::new(Registry::default());
13829        let forwarding = Arc::new(ForwardingTable::default());
13830        let handler =
13831            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13832                .with_health_probe_timeout(Duration::from_secs(5));
13833        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13834        hello_via_sink(
13835            &handler,
13836            &module_ctx,
13837            &mut module_rx,
13838            non_routable_hello_frame_with_control_ops(
13839                "mcp",
13840                300,
13841                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13842            ),
13843        )
13844        .await;
13845        assert!(registry
13846            .get_module("mcp")
13847            .unwrap()
13848            .unwrap()
13849            .manifest
13850            .provides
13851            .is_empty());
13852
13853        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13854        let route_response = handler
13855            .handle_control_frame(
13856                &route_client_ctx,
13857                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13858            )
13859            .await
13860            .unwrap();
13861        assert_eq!(route_response[0].header.ty, FrameType::Error);
13862        assert_eq!(
13863            parse_error(&route_response[0])["code"],
13864            "target_unavailable"
13865        );
13866        assert!(parse_error(&route_response[0])["message"]
13867            .as_str()
13868            .unwrap()
13869            .contains("does not provide the requested target"));
13870        assert!(module_rx.try_recv().is_err());
13871
13872        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13873        let health_handler = handler.clone();
13874        let health_task = tokio::spawn(async move {
13875            health_handler
13876                .handle_control_frame(
13877                    &health_client_ctx,
13878                    supervisor_health_probe_frame(302, "mcp"),
13879                )
13880                .await
13881                .unwrap()
13882        });
13883        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13884            .await
13885            .unwrap()
13886            .unwrap();
13887        assert_eq!(
13888            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13889            ModuleControlRequest::HealthCheck {}
13890        );
13891        handler
13892            .handle_control_frame(
13893                &module_ctx,
13894                health_response(health_frame.header.corr, HealthStatus::Ok),
13895            )
13896            .await
13897            .unwrap();
13898        let health_response = health_task.await.unwrap();
13899        assert_eq!(health_response[0].header.ty, FrameType::Response);
13900        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13901            ClientControlResponse::SupervisorHealthProbe {
13902                module_id, status, ..
13903            } => {
13904                assert_eq!(module_id, "mcp");
13905                assert_eq!(status, HealthStatus::Ok);
13906            }
13907            other => panic!("unexpected health response: {other:?}"),
13908        }
13909
13910        // Exercise the forwarding cleanup path directly while leaving the registry
13911        // advertisement in place. If cleanup leaves a stale control sink behind,
13912        // the next probe will enqueue onto it and wait for the long probe timeout
13913        // instead of returning an immediate no-connection error.
13914        forwarding
13915            .cleanup_connection(module_ctx.connection_id)
13916            .unwrap();
13917        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13918        let cleanup_response = tokio::time::timeout(
13919            Duration::from_millis(200),
13920            handler.handle_control_frame(
13921                &cleanup_probe_ctx,
13922                supervisor_health_probe_frame(303, "mcp"),
13923            ),
13924        )
13925        .await
13926        .expect("probe should fail immediately when the control lane is gone")
13927        .unwrap();
13928        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13929        assert_eq!(
13930            parse_error(&cleanup_response[0])["code"],
13931            "target_unavailable"
13932        );
13933        assert!(parse_error(&cleanup_response[0])["message"]
13934            .as_str()
13935            .unwrap()
13936            .contains("no module connection"));
13937
13938        handler
13939            .cleanup_connection(module_ctx.connection_id)
13940            .unwrap();
13941    }
13942
13943    #[tokio::test]
13944    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13945        let registry = Arc::new(Registry::default());
13946        let supervisor_handle = SupervisorHandle::new();
13947        let supervisor =
13948            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13949                .with_handle(supervisor_handle.clone())
13950                .with_connection_file_path(
13951                    std::env::temp_dir()
13952                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13953                );
13954        let module = supervisor
13955            .supervise_configured(
13956                ModuleSpec {
13957                    carry_containment: false,
13958                    module_id: "warming".to_string(),
13959                    program: fake_aft_stub_path(),
13960                    args: Vec::new(),
13961                    env: Vec::new(),
13962                    reserved: false,
13963                    reserved_prefixes: Vec::new(),
13964                    protocol: ModuleProtocol::Subc,
13965                    overlap: Default::default(),
13966                },
13967                true,
13968            )
13969            .unwrap();
13970        assert_eq!(module.state().unwrap(), ModuleState::Running);
13971
13972        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13973        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13974        let response = handler
13975            .handle_control_frame(
13976                &ctx,
13977                route_open_frame(304, "warming", unique_project_root("warming")),
13978            )
13979            .await
13980            .unwrap();
13981        module.stop().await.unwrap();
13982
13983        assert_eq!(response[0].header.ty, FrameType::Error);
13984        let error = parse_error(&response[0]);
13985        assert_eq!(error["code"], "module_warming");
13986        assert!(error["message"]
13987            .as_str()
13988            .unwrap()
13989            .contains("state=running, enabled=true, live=false"));
13990    }
13991
13992    #[test]
13993    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13994        let handler = ControlHandler::new(Arc::new(Registry::default()));
13995        let capture = EventCapture::default();
13996        let _subscriber =
13997            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13998        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13999        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
14000        let pending = (0..limit).collect::<Vec<_>>();
14001        let response = handler
14002            .route_open_capacity_refusal(
14003                &ctx,
14004                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14005                "busy",
14006                pending.len(),
14007                limit,
14008            )
14009            .unwrap();
14010        assert_eq!(parse_error(&response)["code"], "target_unavailable");
14011        let event = capture
14012            .events()
14013            .into_iter()
14014            .find(|event| {
14015                event.target == "control"
14016                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14017            })
14018            .expect("connection admission refusal event");
14019        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14020        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14021        // Refused before `handle_route_open` parses the request, so the caller
14022        // fields are read from the frame itself.
14023        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14024        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14025    }
14026
14027    #[test]
14028    fn route_open_target_cap_logs_admission_reason_and_capacity() {
14029        let handler = ControlHandler::new(Arc::new(Registry::default()));
14030        let capture = EventCapture::default();
14031        let _subscriber =
14032            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14033        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14034        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14035        let guards = (0..limit)
14036            .map(|_| {
14037                handler
14038                    .route_bind_concurrency
14039                    .try_admit("busy", limit)
14040                    .unwrap()
14041            })
14042            .collect::<Vec<_>>();
14043        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14044            Err(in_flight) => in_flight,
14045            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14046        };
14047        let response = handler
14048            .route_open_target_capacity_refusal(
14049                &ctx,
14050                &RouteOpenCaller::unknown(),
14051                &route_open_frame(397, "busy", unique_project_root("target-cap")),
14052                "busy",
14053                in_flight,
14054            )
14055            .unwrap();
14056        assert_eq!(parse_error(&response)["code"], "target_unavailable");
14057        let event = capture
14058            .events()
14059            .into_iter()
14060            .find(|event| {
14061                event.target == "control"
14062                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14063            })
14064            .expect("target admission refusal event");
14065        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14066        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14067        drop(guards);
14068    }
14069
14070    #[cfg(unix)]
14071    #[tokio::test(start_paused = true)]
14072    async fn supervisor_list_shows_a_failed_modules_next_retry() {
14073        let registry = Arc::new(Registry::default());
14074        let handle = SupervisorHandle::new();
14075        let supervisor =
14076            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14077                .with_handle(handle.clone());
14078        let module = supervisor
14079            .spawn(crate::ModuleSpec {
14080                carry_containment: false,
14081                module_id: "retry-list".into(),
14082                program: PathBuf::from("/bin/sh"),
14083                args: vec!["-c".into(), "exit 17".into()],
14084                env: vec![],
14085                reserved: false,
14086                reserved_prefixes: vec![],
14087                protocol: ModuleProtocol::None,
14088                overlap: Default::default(),
14089            })
14090            .unwrap();
14091        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14092        while module.state().unwrap() != crate::ModuleState::Failed {
14093            assert!(
14094                std::time::Instant::now() < deadline,
14095                "waiting for budget failure"
14096            );
14097            tokio::task::yield_now().await;
14098        }
14099        let handler = ControlHandler::new(registry).with_supervisor(handle);
14100        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14101        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14102        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14103        assert_eq!(body["modules"][0]["state"], "failed");
14104        let retry = body["modules"][0]["next_retry_at_ms"]
14105            .as_u64()
14106            .expect("scheduled retry on the wire");
14107        let now = std::time::SystemTime::now()
14108            .duration_since(std::time::UNIX_EPOCH)
14109            .unwrap()
14110            .as_millis() as u64;
14111        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14112        module.set_enabled(false).await.unwrap();
14113        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14114        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14115        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14116        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14117    }
14118
14119    #[tokio::test]
14120    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14121        let _guard = crate::subtree_resources::fault_read_guard().await;
14122        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14123        let supervisor =
14124            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14125        let module = supervisor
14126            .spawn(crate::ModuleSpec {
14127                carry_containment: false,
14128                module_id: "pid-read-budget".into(),
14129                program: std::env::current_exe().unwrap(),
14130                args: vec![
14131                    "--exact".into(),
14132                    "control::tests::pid_resource_read_fixture".into(),
14133                ],
14134                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14135                    .into_iter()
14136                    .map(|key| (key.into(), dir.display().to_string()))
14137                    .chain(std::iter::once((
14138                        "SUBC_SUBTREE_PID_FIXTURE".into(),
14139                        "1".into(),
14140                    )))
14141                    .collect(),
14142                reserved: false,
14143                reserved_prefixes: Vec::new(),
14144                protocol: ModuleProtocol::None,
14145                overlap: Default::default(),
14146            })
14147            .unwrap();
14148        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14149        while !matches!(
14150            module.child_resource_usage(),
14151            subc_control::ChildResourceUsage::Measured(_)
14152        ) {
14153            assert!(
14154                tokio::time::Instant::now() < ready_deadline,
14155                "fixture process was not measurable"
14156            );
14157            tokio::time::sleep(Duration::from_millis(10)).await;
14158        }
14159        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14160        let reading = crate::subtree_resources::read_process(module.clone()).await;
14161        module.stop().await.unwrap();
14162        assert!(
14163            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14164            "{reading:?}"
14165        );
14166    }
14167
14168    #[test]
14169    fn pid_resource_read_fixture() {
14170        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14171            loop {
14172                std::thread::park_timeout(Duration::from_secs(600));
14173            }
14174        }
14175    }
14176
14177    #[tokio::test]
14178    async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14179        let registry = Arc::new(Registry::default());
14180        let handle = SupervisorHandle::new();
14181        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14182            .with_handle(handle.clone());
14183        supervisor
14184            .supervise_configured(
14185                crate::ModuleSpec {
14186                    module_id: "disabled-resource-reading".into(),
14187                    program: std::env::current_exe().unwrap(),
14188                    args: Vec::new(),
14189                    env: Vec::new(),
14190                    reserved: false,
14191                    reserved_prefixes: Vec::new(),
14192                    protocol: ModuleProtocol::None,
14193                    overlap: Default::default(),
14194                    carry_containment: false,
14195                },
14196                false,
14197            )
14198            .unwrap();
14199        let handler = ControlHandler::new(registry).with_supervisor(handle);
14200        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14201        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14202        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14203        assert!(body["modules"][0].get("tree").is_none());
14204        assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14205    }
14206
14207    #[tokio::test]
14208    async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14209        let _guard = crate::subtree_resources::fault_read_guard().await;
14210        let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14211        let registry = Arc::new(Registry::default());
14212        let handle = SupervisorHandle::new();
14213        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14214            .with_handle(handle.clone());
14215        let module = supervisor
14216            .spawn(crate::ModuleSpec {
14217                carry_containment: false,
14218                module_id: "list-process-priority".into(),
14219                program: std::env::current_exe().unwrap(),
14220                args: vec![
14221                    "--exact".into(),
14222                    "control::tests::pid_resource_read_fixture".into(),
14223                ],
14224                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14225                    .into_iter()
14226                    .map(|key| (key.into(), dir.display().to_string()))
14227                    .chain(std::iter::once((
14228                        "SUBC_SUBTREE_PID_FIXTURE".into(),
14229                        "1".into(),
14230                    )))
14231                    .collect(),
14232                reserved: false,
14233                reserved_prefixes: Vec::new(),
14234                protocol: ModuleProtocol::None,
14235                overlap: Default::default(),
14236            })
14237            .unwrap();
14238        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14239        loop {
14240            let image = module.running_image_agreement().await;
14241            if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14242                break;
14243            }
14244            assert!(
14245                tokio::time::Instant::now() < ready_deadline,
14246                "the fixture process never became observable: {image:?}"
14247            );
14248            tokio::time::sleep(Duration::from_millis(10)).await;
14249        }
14250        let (target, release, _) = crate::subtree_resources::held();
14251        module.set_subtree_target_for_test(target);
14252        let handler = ControlHandler::new(registry).with_supervisor(handle);
14253        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14254        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14255        drop(release);
14256        module.stop().await.unwrap();
14257        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14258        assert_eq!(
14259            body["modules"][0]["resources"]["status"], "measured",
14260            "the optional tree timeout must not discard the process reading: {body}"
14261        );
14262        assert!(body["modules"][0].get("tree").is_none());
14263        assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14264        assert_eq!(
14265            body["modules"][0]["pending_reload"]["image"]["status"], "match",
14266            "the tree timeout must not hide a completed image observation"
14267        );
14268    }
14269
14270    #[cfg(any(target_os = "linux", target_os = "macos"))]
14271    #[tokio::test]
14272    async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14273        let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14274            .with_daemon_provenance(
14275                std::process::id(),
14276                0,
14277                Some(std::env::current_exe().unwrap()),
14278                None,
14279                None,
14280            );
14281        let (probe, release, began) = ExecutableIdentityProbe::held();
14282        handler.daemon_provenance.probe = probe;
14283        let observe = |handler: ControlHandler, corr| {
14284            tokio::spawn(async move {
14285                let (ctx, _) = route_ctx(ConnectionId::new(corr));
14286                let frame =
14287                    Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14288                        .unwrap();
14289                handler
14290                    .handle_supervisor_provenance(&ctx, frame, None)
14291                    .await
14292                    .unwrap()
14293            })
14294        };
14295        let explicit = observe(handler.clone(), 114);
14296        began.await.unwrap();
14297        let concurrent = observe(handler.clone(), 115);
14298        // Keep explicit provenance waiting on its first image observation beyond
14299        // the list deadline; release it only after the concurrent list replies.
14300        tokio::time::sleep(Duration::from_millis(300)).await;
14301        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14302        let list = tokio::time::timeout(
14303            Duration::from_secs(1),
14304            handler.handle_supervisor_list(frame),
14305        )
14306        .await;
14307        let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14308        drop(release);
14309        let replies = explicit.await.unwrap();
14310        let concurrent_replies = concurrent.await.unwrap();
14311        assert!(
14312            explicit_was_pending,
14313            "explicit provenance must await the full observation"
14314        );
14315        let list = list
14316            .expect("a concurrent explicit probe must not stall list")
14317            .unwrap();
14318        let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14319        assert_eq!(list_body["op"], "supervisor.list");
14320        for replies in [replies, concurrent_replies] {
14321            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14322            assert_eq!(
14323                body["daemon"]["daemon_observed"]["running_image"]["status"],
14324                "match"
14325            );
14326        }
14327    }
14328
14329    #[tokio::test]
14330    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14331        let _guard = crate::subtree_resources::fault_read_guard().await;
14332        let registry = Arc::new(Registry::default());
14333        let handle = SupervisorHandle::new();
14334        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14335            .with_handle(handle.clone());
14336        let module = supervisor
14337            .supervise_configured(
14338                crate::ModuleSpec {
14339                    carry_containment: false,
14340                    module_id: "memory-read".into(),
14341                    program: PathBuf::from("disabled-module"),
14342                    args: Vec::new(),
14343                    env: Vec::new(),
14344                    reserved: false,
14345                    reserved_prefixes: Vec::new(),
14346                    protocol: ModuleProtocol::Subc,
14347                    overlap: Default::default(),
14348                },
14349                true,
14350            )
14351            .unwrap();
14352        let (target, release, began) = crate::subtree_resources::held();
14353        // Optional list work may be skipped when its deadline is already spent.
14354        // Establish the stalled fixture before asking the list to observe it.
14355        let stalled_target = target.clone();
14356        let stalled = tokio::task::spawn_blocking(move || {
14357            stalled_target.tree(&subc_os::ProcessTable::default())
14358        });
14359        began.await.unwrap();
14360        module.set_subtree_target_for_test(target);
14361        let handler = ControlHandler::new(registry).with_supervisor(handle);
14362        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14363        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14364        let replies = handler
14365            .handle_client_control_request(
14366                &ctx,
14367                frame,
14368                ClientControlRequest::SupervisorList {},
14369                &mut None,
14370            )
14371            .await
14372            .unwrap();
14373        assert!(
14374            replies.is_empty(),
14375            "list reply must be detached from the connection reader"
14376        );
14377        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14378        let replies = handler
14379            .handle_client_control_request(
14380                &ctx,
14381                frame,
14382                ClientControlRequest::SupervisorSpawnSnapshot {},
14383                &mut None,
14384            )
14385            .await
14386            .unwrap();
14387        assert_eq!(
14388            replies.len(),
14389            1,
14390            "another request must complete while the subtree read stalls"
14391        );
14392        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14393            .await
14394            .unwrap()
14395            .unwrap();
14396        release.send(()).unwrap();
14397        drop(release);
14398        stalled.await.unwrap();
14399        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14400        assert!(
14401            body["modules"][0].get("tree").is_none(),
14402            "an unknown tree reading must be omitted after the bounded read expires"
14403        );
14404        // Lists export completed work rather than measuring synchronously.
14405        // Complete a refresh after the fixture is released before asserting
14406        // that a later list contains its measured tree.
14407        handler
14408            .resource_tracker
14409            .read(handler.supervisor.list(), false, "released_tree_fixture")
14410            .await;
14411        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14412        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14413        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14414        assert_eq!(
14415            body["modules"][0]["tree"]["source"], "process_tree",
14416            "the next request must export the completed refresh"
14417        );
14418        assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14419    }
14420
14421    #[tokio::test]
14422    async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
14423        use subc_control::{
14424            AgentCommandResources, ChildMemoryKind, ChildResourceReading,
14425            ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
14426            TreeReading, TreeSource,
14427        };
14428        let registry = Arc::new(Registry::default());
14429        let handle = SupervisorHandle::new();
14430        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14431            .with_handle(handle.clone());
14432        supervisor
14433            .supervise_configured(
14434                crate::ModuleSpec {
14435                    carry_containment: false,
14436                    module_id: "budget-worker".into(),
14437                    program: PathBuf::from("disabled-module"),
14438                    args: Vec::new(),
14439                    env: Vec::new(),
14440                    reserved: false,
14441                    reserved_prefixes: Vec::new(),
14442                    protocol: ModuleProtocol::Subc,
14443                    overlap: Default::default(),
14444                },
14445                true,
14446            )
14447            .unwrap();
14448        let handler = ControlHandler::new(registry).with_supervisor(handle);
14449        let now = u64::try_from(
14450            std::time::SystemTime::now()
14451                .duration_since(std::time::UNIX_EPOCH)
14452                .unwrap()
14453                .as_millis(),
14454        )
14455        .unwrap();
14456        let measured = now - 1_000;
14457        let hour = CpuLastHour::new(100, 60_000);
14458        let daemon = DaemonResources::new(7, 1, "test")
14459            .with_measured_at_ms(Some(measured))
14460            .with_resources(Some(ChildResourceUsage::Measured(
14461                ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
14462            )))
14463            .with_cpu_last_hour(Some(hour.clone()));
14464        let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
14465            .with_measured_at_ms(Some(measured))
14466            .with_cpu_last_hour(Some(hour.clone()));
14467        let tree = TreeReading::new(
14468            150,
14469            50,
14470            ChildMemoryKind::ResidentSet,
14471            1,
14472            TreeSource::ProcessTree,
14473        )
14474        .with_measured_at_ms(Some(measured));
14475        handler.resource_tracker.seed_committed_for_test(
14476            &handler.supervisor.list(),
14477            crate::resource_totals::Readings {
14478                modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
14479                daemon: Some(daemon),
14480                agent_commands: Some(vec![agents]),
14481                ..Default::default()
14482            },
14483            now,
14484        );
14485        list_diagnostics::scope(async {
14486            tokio::time::sleep(Duration::from_millis(110)).await;
14487            assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
14488            let request =
14489                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14490            let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
14491            let ClientControlResponse::SupervisorList {
14492                modules,
14493                daemon,
14494                agent_commands,
14495                ..
14496            } = serde_json::from_slice(&replies[0].body).unwrap()
14497            else {
14498                panic!("expected a list response")
14499            };
14500            assert!(matches!(
14501                modules[0].resources,
14502                Some(ChildResourceUsage::Unavailable {
14503                    reason: ChildResourceUnavailableReason::Unreadable
14504                })
14505            ));
14506            assert_eq!(
14507                (
14508                    modules[0].tree.as_ref().unwrap().cpu_ms,
14509                    modules[0].tree.as_ref().unwrap().measured_at_ms
14510                ),
14511                (150, Some(measured))
14512            );
14513            let daemon = daemon.unwrap();
14514            let agents = agent_commands.unwrap();
14515            assert_eq!(daemon.measured_at_ms, Some(measured));
14516            assert_eq!(
14517                (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
14518                (1, 50, Some(measured))
14519            );
14520            for hour in [
14521                modules[0].cpu_last_hour.as_ref(),
14522                daemon.cpu_last_hour.as_ref(),
14523                agents[0].cpu_last_hour.as_ref(),
14524            ] {
14525                let hour = hour.unwrap();
14526                assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
14527            }
14528        })
14529        .await;
14530    }
14531
14532    #[tokio::test]
14533    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14534        let registry = Arc::new(Registry::default());
14535        let handle = SupervisorHandle::new();
14536        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14537            .with_handle(handle.clone());
14538        supervisor
14539            .supervise_configured(
14540                crate::ModuleSpec {
14541                    carry_containment: false,
14542                    module_id: "drift".into(),
14543                    program: PathBuf::from("disabled-module"),
14544                    args: Vec::new(),
14545                    env: Vec::new(),
14546                    reserved: false,
14547                    reserved_prefixes: Vec::new(),
14548                    protocol: ModuleProtocol::Subc,
14549                    overlap: Default::default(),
14550                },
14551                false,
14552            )
14553            .unwrap();
14554        let handler = ControlHandler::new(registry).with_supervisor(handle);
14555        for configured in [None, Some(false), Some(true)] {
14556            handler.capability_evaluator.configure(
14557                configured.map(|enabled| ("drift".to_string(), enabled)),
14558                BTreeMap::new(),
14559            );
14560            let frame =
14561                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14562            let replies = handler
14563                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14564                .await
14565                .unwrap();
14566            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14567            assert!(body.get("configured_enabled").is_none());
14568            assert_eq!(body["op"], "supervisor.ack");
14569            let frame =
14570                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14571            let replies = handler.handle_supervisor_list(frame).await.unwrap();
14572            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14573            assert_eq!(
14574                body["modules"][0].get("configured_enabled"),
14575                configured.map(Value::Bool).as_ref()
14576            );
14577            assert!(body["modules"][0].get("tree").is_none());
14578            assert!(body["modules"][0].get("subtree").is_none());
14579        }
14580    }
14581
14582    #[tokio::test(flavor = "current_thread")]
14583    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14584        let handler = ControlHandler::new(Arc::new(Registry::default()));
14585        let capture = EventCapture::default();
14586        let _subscriber =
14587            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14588        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14589        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14590
14591        let _ = handler
14592            .handle_client_control_request(
14593                &ctx,
14594                frame,
14595                ClientControlRequest::SupervisorSetEnabled {
14596                    module_id: "broca".to_string(),
14597                    enabled: false,
14598                },
14599                &mut None,
14600            )
14601            .await
14602            .unwrap();
14603
14604        let events = capture
14605            .events()
14606            .into_iter()
14607            .filter(|event| {
14608                event.target == "control"
14609                    && event.fields.get("message").map(String::as_str)
14610                        == Some("supervisor request received")
14611            })
14612            .collect::<Vec<_>>();
14613        assert_eq!(events.len(), 1, "one supervisor request log line");
14614        let event = &events[0];
14615        assert_eq!(event.level, tracing::Level::INFO);
14616        assert_eq!(
14617            event.fields.get("op"),
14618            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14619        );
14620        assert_eq!(
14621            event.fields.get("module_id"),
14622            Some(&"\"broca\"".to_string())
14623        );
14624        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14625        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14626        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14627    }
14628
14629    #[tokio::test(flavor = "current_thread")]
14630    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14631        let registry = Arc::new(Registry::default());
14632        let module_connection = ConnectionId::new(303);
14633        registry
14634            .register_with_control_ops(
14635                manifest("aft", PROTOCOL_VERSION),
14636                PROTOCOL_VERSION,
14637                module_connection,
14638                module_baseline_control_ops(),
14639            )
14640            .unwrap();
14641        let handler = ControlHandler::new(registry);
14642        let capture = EventCapture::default();
14643        let _subscriber =
14644            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14645        let (ctx, _rx) = route_ctx(module_connection);
14646        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14647
14648        let _ = handler
14649            .handle_client_control_request(
14650                &ctx,
14651                frame,
14652                ClientControlRequest::SupervisorSetEnabled {
14653                    module_id: "broca".to_string(),
14654                    enabled: false,
14655                },
14656                &mut None,
14657            )
14658            .await
14659            .unwrap();
14660
14661        let callers = capture
14662            .events()
14663            .into_iter()
14664            .filter(|event| {
14665                event.fields.get("message").map(String::as_str)
14666                    == Some("supervisor request received")
14667            })
14668            .map(|event| event.fields.get("caller").cloned())
14669            .collect::<Vec<_>>();
14670        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14671    }
14672
14673    #[tokio::test(flavor = "current_thread")]
14674    async fn supervisor_list_does_not_log_request_received() {
14675        let handler = ControlHandler::new(Arc::new(Registry::default()));
14676        let capture = EventCapture::default();
14677        let _subscriber =
14678            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14679        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14680        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14681
14682        handler
14683            .handle_client_control_request(
14684                &ctx,
14685                frame,
14686                ClientControlRequest::SupervisorList {},
14687                &mut None,
14688            )
14689            .await
14690            .unwrap();
14691
14692        assert!(capture.events().into_iter().all(|event| {
14693            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14694        }));
14695    }
14696
14697    #[tokio::test(flavor = "current_thread")]
14698    async fn supervisor_rescan_preview_does_not_log_request_received() {
14699        let handler = ControlHandler::new(Arc::new(Registry::default()));
14700        let capture = EventCapture::default();
14701        let _subscriber =
14702            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14703        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14704        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14705
14706        handler
14707            .handle_client_control_request(
14708                &ctx,
14709                frame,
14710                ClientControlRequest::SupervisorRescan { preview: true },
14711                &mut None,
14712            )
14713            .await
14714            .unwrap();
14715
14716        assert!(capture.events().into_iter().all(|event| {
14717            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14718        }));
14719    }
14720
14721    struct ApproveOperator;
14722    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14723        fn prompt(
14724            &self,
14725            _: &str,
14726            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14727        ) -> crate::operator_confirm::ProviderResult {
14728            crate::operator_confirm::ProviderResult::Approved
14729        }
14730    }
14731
14732    #[tokio::test]
14733    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14734        assert_eq!(
14735            ControlHandler::default().privileged_operator_prompt,
14736            PrivilegedOperatorPrompt::Log
14737        );
14738        for (initial, changed, running) in [
14739            ("log", "enforce", PrivilegedOperatorPrompt::Log),
14740            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14741        ] {
14742            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14743            let path = dir.join("subc.jsonc");
14744            std::fs::write(
14745                &path,
14746                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14747            )
14748            .unwrap();
14749            let config = crate::daemon_config::load(&path).unwrap().unwrap();
14750            let registry = Arc::new(Registry::default());
14751            let handle = SupervisorHandle::new();
14752            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14753                .with_handle(handle.clone());
14754            let handler = ControlHandler::new(registry)
14755                .with_supervisor(handle)
14756                .with_privileged_operator_prompt(config.privileged_operator_prompt)
14757                .with_supervisor_rescan(supervisor, &path, config.port);
14758            handler.forwarding.operator_confirms().configure(
14759                Arc::new(ApproveOperator),
14760                crate::operator_confirm::OperatorLimits::default(),
14761            );
14762            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14763
14764            for (disk_mode, needs_restart) in [
14765                (Some(initial), false),
14766                (Some(changed), true),
14767                (Some(changed), true),
14768                (None, running == PrivilegedOperatorPrompt::Enforce),
14769                (Some(initial), false),
14770            ] {
14771                let mut doc = json!({"version":1});
14772                if let Some(mode) = disk_mode {
14773                    doc["privileged_operator_prompt"] = json!(mode);
14774                }
14775                std::fs::write(&path, doc.to_string()).unwrap();
14776                for preview in [true, false] {
14777                    let request = ClientControlRequest::SupervisorRescan { preview };
14778                    let frame = Frame::build(
14779                        FrameType::Request,
14780                        control_flags(),
14781                        0,
14782                        0,
14783                        1,
14784                        serde_json::to_vec(&request).unwrap(),
14785                    )
14786                    .unwrap();
14787                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14788                    assert_eq!(frames.len(), 1);
14789                    let ClientControlResponse::SupervisorRescan { result } =
14790                        serde_json::from_slice(&frames[0].body).unwrap()
14791                    else {
14792                        panic!("expected a rescan response");
14793                    };
14794                    let expected = if needs_restart {
14795                        vec!["privileged_operator_prompt"]
14796                    } else {
14797                        Vec::new()
14798                    };
14799                    assert_eq!(result.restart_required, expected);
14800                    assert_eq!(handler.privileged_operator_prompt, running);
14801                    assert_eq!(
14802                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14803                        running
14804                    );
14805                }
14806            }
14807        }
14808    }
14809
14810    #[tokio::test]
14811    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14812        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14813        let path = dir.join("subc.jsonc");
14814        let registry = Arc::new(Registry::default());
14815        let handle = SupervisorHandle::new();
14816        let supervisor =
14817            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14818        let handler = ControlHandler::new(registry)
14819            .with_supervisor(handle)
14820            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14821            .with_supervisor_rescan(supervisor, &path, None);
14822        handler.forwarding.operator_confirms().configure(
14823            Arc::new(ApproveOperator),
14824            crate::operator_confirm::OperatorLimits::default(),
14825        );
14826        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14827        for preview in [true, false] {
14828            std::fs::write(
14829                &path,
14830                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14831            )
14832            .unwrap();
14833            let request = ClientControlRequest::SupervisorRescan { preview };
14834            let frame = Frame::build(
14835                FrameType::Request,
14836                control_flags(),
14837                0,
14838                0,
14839                1,
14840                serde_json::to_vec(&request).unwrap(),
14841            )
14842            .unwrap();
14843            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14844            assert_eq!(frames.len(), 1);
14845            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14846            assert_eq!(
14847                handler.privileged_operator_prompt,
14848                PrivilegedOperatorPrompt::Enforce
14849            );
14850        }
14851    }
14852
14853    /// One wire code has several senders, so the refusal line names the check
14854    /// that refused. This drives the shared refusal path for ordinary refusals
14855    /// with an unregistered
14856    /// target and requires the branch label on the event.
14857    #[tokio::test]
14858    async fn route_open_refusal_names_the_check_that_refused() {
14859        let handler = ControlHandler::new(Arc::new(Registry::default()));
14860        let capture = EventCapture::default();
14861        let _subscriber =
14862            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14863        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14864        let response = handler
14865            .handle_control_frame(
14866                &ctx,
14867                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14868            )
14869            .await
14870            .unwrap();
14871
14872        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14873        let event = capture
14874            .events()
14875            .into_iter()
14876            .find(|event| {
14877                event.target == "control"
14878                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14879            })
14880            .expect("route.open refusal event");
14881        assert_eq!(
14882            event.fields.get("reason"),
14883            Some(&"\"not_registered\"".to_string())
14884        );
14885    }
14886
14887    #[tokio::test]
14888    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14889        let registry = Arc::new(Registry::default());
14890        let supervisor_handle = SupervisorHandle::new();
14891        let supervisor =
14892            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14893                .with_handle(supervisor_handle.clone())
14894                .with_connection_file_path(std::env::temp_dir().join(format!(
14895                    "subc-route-open-refusal-info-{}",
14896                    std::process::id()
14897                )));
14898        let module = supervisor
14899            .supervise_configured(
14900                ModuleSpec {
14901                    carry_containment: false,
14902                    module_id: "warming".to_string(),
14903                    program: fake_aft_stub_path(),
14904                    args: Vec::new(),
14905                    env: Vec::new(),
14906                    reserved: false,
14907                    reserved_prefixes: Vec::new(),
14908                    protocol: ModuleProtocol::Subc,
14909                    overlap: Default::default(),
14910                },
14911                true,
14912            )
14913            .unwrap();
14914        assert_eq!(module.state().unwrap(), ModuleState::Running);
14915
14916        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14917        assert!(handler
14918            .counters()
14919            .snapshot()
14920            .get("route_open_refused_by_code")
14921            .is_none());
14922        let capture = EventCapture::default();
14923        let _subscriber =
14924            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14925        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14926        let response = handler
14927            .handle_control_frame(
14928                &ctx,
14929                route_open_frame(394, "warming", unique_project_root("refusal-info")),
14930            )
14931            .await
14932            .unwrap();
14933        module.stop().await.unwrap();
14934
14935        assert_eq!(parse_error(&response[0])["code"], "module_warming");
14936        let event = capture
14937            .events()
14938            .into_iter()
14939            .find(|event| {
14940                event.target == "control"
14941                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14942            })
14943            .expect("route.open refusal event");
14944        assert_eq!(
14945            event.fields.get("module_id"),
14946            Some(&"\"warming\"".to_string())
14947        );
14948        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14949        assert_eq!(
14950            event.fields.get("reason"),
14951            Some(&"\"supervised_not_registered\"".to_string())
14952        );
14953        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14954        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14955        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14956        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14957        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14958        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14959        assert_eq!(
14960            handler.counters().snapshot()["route_open_refused_by_code"],
14961            json!({ "module_warming": 1 })
14962        );
14963    }
14964
14965    const OUTAGE_START: &str = "route.open refusing module: not serving";
14966    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14967
14968    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14969        capture
14970            .events()
14971            .into_iter()
14972            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14973            .collect()
14974    }
14975
14976    fn supervise_stub(
14977        registry: &Arc<Registry>,
14978        module_id: &str,
14979        enabled: bool,
14980    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14981        let supervisor_handle = SupervisorHandle::new();
14982        let supervisor =
14983            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14984                .with_handle(supervisor_handle.clone())
14985                .with_connection_file_path(std::env::temp_dir().join(format!(
14986                    "subc-route-outage-{module_id}-{}",
14987                    std::process::id()
14988                )));
14989        let module = supervisor
14990            .supervise_configured(
14991                ModuleSpec {
14992                    carry_containment: false,
14993                    module_id: module_id.to_string(),
14994                    program: fake_aft_stub_path(),
14995                    args: Vec::new(),
14996                    env: Vec::new(),
14997                    reserved: false,
14998                    reserved_prefixes: Vec::new(),
14999                    protocol: ModuleProtocol::Subc,
15000                    overlap: Default::default(),
15001                },
15002                enabled,
15003            )
15004            .unwrap();
15005        (supervisor_handle, module)
15006    }
15007
15008    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
15009        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
15010            module_id: module_id.to_string(),
15011            drain_timeout_ms: Some(50),
15012        })
15013        .unwrap();
15014        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
15015    }
15016
15017    /// Two handlers built over one forwarding table must share one outage
15018    /// tracker; separate trackers would each log their own opening line for
15019    /// the same outage.
15020    #[test]
15021    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
15022        let registry = Arc::new(Registry::default());
15023        let forwarding = Arc::new(ForwardingTable::default());
15024        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15025        let second = ControlHandler::with_forwarding(registry, forwarding);
15026        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
15027    }
15028
15029    /// A client can name any module id it likes. Refusing an unknown one,
15030    /// however often, must not create outage state or outage lines, or the
15031    /// tracker would be a memory sink any client could fill.
15032    #[tokio::test(flavor = "current_thread")]
15033    async fn route_open_unknown_module_refusals_add_no_outage_state() {
15034        let handler = ControlHandler::new(Arc::new(Registry::default()));
15035        let capture = EventCapture::default();
15036        let _subscriber =
15037            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15038        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
15039        for corr in 0..8 {
15040            let response = handler
15041                .handle_control_frame(
15042                    &ctx,
15043                    route_open_frame(
15044                        380 + corr,
15045                        &format!("nobody-{corr}"),
15046                        unique_project_root("outage-unknown"),
15047                    ),
15048                )
15049                .await
15050                .unwrap();
15051            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15052        }
15053
15054        assert_eq!(handler.route_outages.tracked_module_count(), 0);
15055        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
15056        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
15057    }
15058
15059    /// Drives the refusal path end to end: a supervised module that served
15060    /// before and stopped being registered with no instruction to stop is a
15061    /// WARN, and the same module refused after an operator `supervisor.restart`
15062    /// is an INFO.
15063    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15064    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
15065        let registry = Arc::new(Registry::default());
15066        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
15067        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15068        let capture = EventCapture::default();
15069        let _subscriber =
15070            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15071        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
15072        // The stub never registers, so pretend it served once: otherwise every
15073        // refusal would fall in its startup window.
15074        handler.route_outages.record_accepted("outage-restart");
15075
15076        let response = handler
15077            .handle_control_frame(
15078                &ctx,
15079                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
15080            )
15081            .await
15082            .unwrap();
15083        assert_eq!(response[0].header.ty, FrameType::Error);
15084        let starts = outage_lines(&capture, OUTAGE_START);
15085        assert_eq!(starts.len(), 1, "{starts:?}");
15086        assert_eq!(starts[0].level, tracing::Level::WARN);
15087        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
15088        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
15089        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
15090        handler.route_outages.record_accepted("outage-restart");
15091        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
15092
15093        let restart = handler
15094            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
15095            .await
15096            .unwrap();
15097        assert_eq!(
15098            restart[0].header.ty,
15099            FrameType::Response,
15100            "{:?}",
15101            parse_error(&restart[0])
15102        );
15103        handler
15104            .handle_control_frame(
15105                &ctx,
15106                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
15107            )
15108            .await
15109            .unwrap();
15110        module.stop().await.unwrap();
15111
15112        let starts = outage_lines(&capture, OUTAGE_START);
15113        assert_eq!(starts.len(), 2, "{starts:?}");
15114        assert_eq!(starts[1].level, tracing::Level::INFO);
15115        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
15116    }
15117
15118    /// A restart refused before it touched the module (here: the module is
15119    /// disabled) must clear its operator mark, so the next real outage is
15120    /// still reported as a warning.
15121    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15122    async fn failed_operator_restart_leaves_no_operator_mark() {
15123        let registry = Arc::new(Registry::default());
15124        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
15125        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15126        let capture = EventCapture::default();
15127        let _subscriber =
15128            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15129        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15130        handler.route_outages.record_accepted("outage-disabled");
15131
15132        let restart = handler
15133            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15134            .await
15135            .unwrap();
15136        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15137        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15138
15139        handler
15140            .handle_control_frame(
15141                &ctx,
15142                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15143            )
15144            .await
15145            .unwrap();
15146        let starts = outage_lines(&capture, OUTAGE_START);
15147        assert_eq!(starts.len(), 1, "{starts:?}");
15148        assert_eq!(starts[0].level, tracing::Level::WARN);
15149    }
15150
15151    #[tokio::test(flavor = "current_thread")]
15152    async fn route_open_unknown_module_escapes_target_module_id() {
15153        let handler = ControlHandler::new(Arc::new(Registry::default()));
15154        let capture = EventCapture::default();
15155        let _subscriber =
15156            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15157        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15158        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15159        let response = handler
15160            .handle_control_frame(
15161                &ctx,
15162                route_open_frame(
15163                    395,
15164                    hostile_module_id,
15165                    unique_project_root("hostile-target-module-id"),
15166                ),
15167            )
15168            .await
15169            .unwrap();
15170
15171        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15172        let event = capture
15173            .events()
15174            .into_iter()
15175            .find(|event| {
15176                event.target == "control"
15177                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15178            })
15179            .expect("route.open unknown-module refusal event");
15180        let logged = event.fields.get("module_id").expect("module_id field");
15181        assert!(!logged.bytes().any(|byte| byte < 0x20));
15182        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15183    }
15184
15185    /// A route.open refusal is the daemon's own answer, so its ERROR carries
15186    /// the daemon-origin bit on the frame the client receives.
15187    #[tokio::test(flavor = "current_thread")]
15188    async fn route_open_refusal_carries_daemon_origin() {
15189        let handler = ControlHandler::new(Arc::new(Registry::default()));
15190        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15191        let response = handler
15192            .handle_control_frame(
15193                &ctx,
15194                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15195            )
15196            .await
15197            .unwrap();
15198
15199        assert_eq!(response.len(), 1);
15200        assert_eq!(response[0].header.ty, FrameType::Error);
15201        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15202        assert!(
15203            response[0].header.flags.is_daemon_origin(),
15204            "route.open refusal must carry daemon origin: {:?}",
15205            response[0].header.flags
15206        );
15207    }
15208
15209    /// The drain `module_reloading` refusal is daemon-synthesized for an
15210    /// endpoint that is reloading, so it carries the daemon-origin bit on the
15211    /// frame the client receives.
15212    #[tokio::test(flavor = "current_thread")]
15213    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15214        let registry = Arc::new(Registry::default());
15215        let forwarding = Arc::new(ForwardingTable::default());
15216        let handler =
15217            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15218        let module_connection = ConnectionId::new(95);
15219        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15220        hello_via_sink(
15221            &handler,
15222            &module_ctx,
15223            &mut module_rx,
15224            hello_frame("aft", PROTOCOL_VERSION, 395),
15225        )
15226        .await;
15227        forwarding
15228            .begin_module_drain("aft", RouteCloseReason::Restart)
15229            .unwrap()
15230            .expect("the module endpoint drains");
15231
15232        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15233        let response = handler
15234            .handle_control_frame(
15235                &client_ctx,
15236                route_open_frame(396, "aft", unique_project_root("drain-origin")),
15237            )
15238            .await
15239            .unwrap();
15240
15241        assert_eq!(response.len(), 1);
15242        assert_eq!(response[0].header.ty, FrameType::Error);
15243        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15244        assert!(
15245            response[0].header.flags.is_daemon_origin(),
15246            "drain module_reloading must carry daemon origin: {:?}",
15247            response[0].header.flags
15248        );
15249    }
15250
15251    /// A module that refuses a bind with a daemon code still cannot mint a
15252    /// daemon refusal: the relayed `module_reloading` body reaches the client
15253    /// with the daemon-origin bit clear.
15254    #[tokio::test(flavor = "current_thread")]
15255    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15256        let registry = Arc::new(Registry::default());
15257        let forwarding = Arc::new(ForwardingTable::default());
15258        let handler =
15259            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15260        let module_connection = ConnectionId::new(95);
15261        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15262        hello_via_sink(
15263            &handler,
15264            &module_ctx,
15265            &mut module_rx,
15266            hello_frame("aft", PROTOCOL_VERSION, 395),
15267        )
15268        .await;
15269
15270        let client_connection = ConnectionId::new(96);
15271        let (client_ctx, _client_rx) = route_ctx(client_connection);
15272        let (route_task, bind) = relay_route_open(
15273            &handler,
15274            client_connection,
15275            &client_ctx.egress,
15276            &mut module_rx,
15277            396,
15278            "aft",
15279            "spoofed-reloading",
15280        )
15281        .await;
15282        // The module sets the origin bit on its own rejection too: neither
15283        // the bit nor the daemon code may survive the relay as daemon-owned.
15284        let rejection = Frame::build(
15285            FrameType::Error,
15286            crate::router::daemon_error_flags(),
15287            0,
15288            0,
15289            bind.header.corr,
15290            serde_json::to_vec(&ErrorBody::new(
15291                "module_reloading",
15292                "module refused route.bind",
15293            ))
15294            .unwrap(),
15295        )
15296        .unwrap();
15297        handler
15298            .handle_control_frame(&module_ctx, rejection)
15299            .await
15300            .unwrap();
15301
15302        let response = route_task.await.unwrap();
15303        assert_eq!(response.len(), 1);
15304        assert_eq!(response[0].header.ty, FrameType::Error);
15305        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15306        assert!(
15307            !response[0].header.flags.is_daemon_origin(),
15308            "a module's own rejection must not carry daemon origin: {:?}",
15309            response[0].header.flags
15310        );
15311    }
15312
15313    #[tokio::test(flavor = "current_thread")]
15314    async fn route_open_module_rejection_uses_daemon_counter_key() {
15315        let registry = Arc::new(Registry::default());
15316        let forwarding = Arc::new(ForwardingTable::default());
15317        let handler =
15318            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15319        let module_connection = ConnectionId::new(95);
15320        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15321        hello_via_sink(
15322            &handler,
15323            &module_ctx,
15324            &mut module_rx,
15325            hello_frame("aft", PROTOCOL_VERSION, 395),
15326        )
15327        .await;
15328
15329        let client_connection = ConnectionId::new(96);
15330        let (client_ctx, _client_rx) = route_ctx(client_connection);
15331        let capture = EventCapture::default();
15332        let _subscriber =
15333            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15334        let (route_task, bind) = relay_route_open(
15335            &handler,
15336            client_connection,
15337            &client_ctx.egress,
15338            &mut module_rx,
15339            396,
15340            "aft",
15341            "hostile-module-code",
15342        )
15343        .await;
15344        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15345        let rejection = Frame::build(
15346            FrameType::Error,
15347            control_flags(),
15348            0,
15349            0,
15350            bind.header.corr,
15351            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15352        )
15353        .unwrap();
15354        handler
15355            .handle_control_frame(&module_ctx, rejection)
15356            .await
15357            .unwrap();
15358
15359        let response = route_task.await.unwrap();
15360        assert_eq!(parse_error(&response[0])["code"], hostile_code);
15361        let counters = handler.counters().snapshot();
15362        assert_eq!(
15363            counters["route_open_refused_by_code"],
15364            json!({ "module_rejected": 1 })
15365        );
15366        assert!(counters["route_open_refused_by_code"]
15367            .get(hostile_code)
15368            .is_none());
15369
15370        let event = capture
15371            .events()
15372            .into_iter()
15373            .find(|event| {
15374                event.target == "control"
15375                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15376            })
15377            .expect("route.open module-rejection refusal event");
15378        let logged = event.fields.get("module_code").expect("module_code field");
15379        assert!(!logged.bytes().any(|byte| byte < 0x20));
15380        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15381    }
15382
15383    #[tokio::test]
15384    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15385        let registry = Arc::new(Registry::default());
15386        let supervisor_handle = SupervisorHandle::new();
15387        let missing_program = std::env::temp_dir().join(format!(
15388            "subc-route-open-missing-program-{}",
15389            std::process::id()
15390        ));
15391        let supervisor =
15392            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
15393                .with_handle(supervisor_handle.clone());
15394        let module = supervisor
15395            .supervise_configured(
15396                ModuleSpec {
15397                    carry_containment: false,
15398                    module_id: "failed".to_string(),
15399                    program: missing_program,
15400                    args: Vec::new(),
15401                    env: Vec::new(),
15402                    reserved: false,
15403                    reserved_prefixes: Vec::new(),
15404                    protocol: ModuleProtocol::Subc,
15405                    overlap: Default::default(),
15406                },
15407                true,
15408            )
15409            .unwrap();
15410        assert_eq!(module.state().unwrap(), ModuleState::Failed);
15411
15412        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15413        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15414        let response = handler
15415            .handle_control_frame(
15416                &ctx,
15417                route_open_frame(305, "failed", unique_project_root("failed")),
15418            )
15419            .await
15420            .unwrap();
15421
15422        assert_eq!(response[0].header.ty, FrameType::Error);
15423        let error = parse_error(&response[0]);
15424        assert_eq!(error["code"], "target_unavailable");
15425        assert!(error["message"]
15426            .as_str()
15427            .unwrap()
15428            .contains("state=failed, enabled=true, live=false"));
15429    }
15430
15431    #[tokio::test]
15432    async fn route_open_role_mismatch_remains_target_unavailable() {
15433        let registry = Arc::new(Registry::default());
15434        let handler = ControlHandler::new(Arc::clone(&registry));
15435        handler
15436            .handle_control(
15437                ConnectionId::new(41),
15438                non_routable_hello_frame_with_control_ops("health-only", 306, None),
15439            )
15440            .unwrap();
15441
15442        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15443        let response = handler
15444            .handle_control_frame(
15445                &ctx,
15446                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15447            )
15448            .await
15449            .unwrap();
15450
15451        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15452        assert!(parse_error(&response[0])["message"]
15453            .as_str()
15454            .unwrap()
15455            .contains("does not provide the requested target"));
15456    }
15457
15458    #[tokio::test]
15459    async fn route_open_inactive_registration_remains_target_unavailable() {
15460        let registry = Arc::new(Registry::default());
15461        let handler = ControlHandler::new(Arc::clone(&registry));
15462        handler
15463            .handle_control(
15464                ConnectionId::new(43),
15465                hello_frame("inactive", PROTOCOL_VERSION, 308),
15466            )
15467            .unwrap();
15468        assert!(registry
15469            .set_module_state_for_test("inactive", ChannelState::Closed)
15470            .unwrap());
15471
15472        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15473        let response = handler
15474            .handle_control_frame(
15475                &ctx,
15476                route_open_frame(309, "inactive", unique_project_root("inactive")),
15477            )
15478            .await
15479            .unwrap();
15480
15481        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15482        assert!(parse_error(&response[0])["message"]
15483            .as_str()
15484            .unwrap()
15485            .contains("is not active"));
15486    }
15487
15488    #[tokio::test]
15489    async fn late_health_reply_is_recorded_through_the_module_response_path() {
15490        let registry = Arc::new(Registry::default());
15491        let forwarding = Arc::new(ForwardingTable::default());
15492        let supervisor_handle = SupervisorHandle::new();
15493        let supervisor =
15494            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
15495                .with_forwarding(Arc::clone(&forwarding))
15496                .with_handle(supervisor_handle.clone());
15497        let module = supervisor
15498            .supervise_configured(
15499                crate::ModuleSpec {
15500                    carry_containment: false,
15501                    module_id: "late-health-response".to_string(),
15502                    program: PathBuf::from("disabled-module"),
15503                    args: Vec::new(),
15504                    env: Vec::new(),
15505                    reserved: false,
15506                    reserved_prefixes: Vec::new(),
15507                    protocol: ModuleProtocol::Subc,
15508                    overlap: Default::default(),
15509                },
15510                false,
15511            )
15512            .unwrap();
15513        let handler =
15514            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15515                .with_supervisor(supervisor_handle);
15516        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15517        handler
15518            .handle_control_frame(
15519                &module_ctx,
15520                hello_frame_with_control_ops(
15521                    "late-health-response",
15522                    PROTOCOL_VERSION,
15523                    7,
15524                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15525                ),
15526            )
15527            .await
15528            .unwrap();
15529        let probe_started_at = Instant::now() - Duration::from_millis(80);
15530        let pending = forwarding
15531            .begin_health_probe_rpc_for(
15532                "late-health-response",
15533                MODULE_CONTROL_OP_HEALTH_CHECK,
15534                probe_started_at,
15535                Instant::now() - Duration::from_millis(1),
15536            )
15537            .unwrap();
15538        assert!(forwarding
15539            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15540            .unwrap());
15541
15542        let responses = handler
15543            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15544            .await
15545            .unwrap();
15546
15547        assert!(responses.is_empty());
15548        let health = module.status().unwrap().health;
15549        assert_eq!(health.late_answer_count, 1);
15550        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15551    }
15552
15553    #[tokio::test]
15554    async fn health_probe_timeout_and_module_death_are_typed() {
15555        let registry = Arc::new(Registry::default());
15556        let forwarding = Arc::new(ForwardingTable::default());
15557        let handler =
15558            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15559                .with_health_probe_timeout(Duration::from_millis(50));
15560        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15561        hello_via_sink(
15562            &handler,
15563            &module_ctx,
15564            &mut module_rx,
15565            hello_frame_with_control_ops(
15566                "aft",
15567                PROTOCOL_VERSION,
15568                7,
15569                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15570            ),
15571        )
15572        .await;
15573
15574        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15575        let responses = handler
15576            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15577            .await
15578            .unwrap();
15579        assert_eq!(responses[0].header.ty, FrameType::Error);
15580        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15581        let _ = module_rx.try_recv();
15582
15583        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15584        let health_handler = handler.clone();
15585        let death_task = tokio::spawn(async move {
15586            health_handler
15587                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15588                .await
15589                .unwrap()
15590        });
15591        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15592            .await
15593            .unwrap()
15594            .unwrap();
15595        handler
15596            .cleanup_connection(module_ctx.connection_id)
15597            .unwrap();
15598        let responses = death_task.await.unwrap();
15599        assert_eq!(responses[0].header.ty, FrameType::Error);
15600        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15601    }
15602
15603    #[test]
15604    fn hello_requires_exact_protocol_version() {
15605        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15606            let registry = Arc::new(Registry::default());
15607            let handler = ControlHandler::new(Arc::clone(&registry));
15608            let responses = handler
15609                .handle_control(
15610                    ConnectionId::new(connection),
15611                    hello_frame("aft", offered, 9),
15612                )
15613                .unwrap();
15614
15615            assert_eq!(responses.len(), 1);
15616            assert_eq!(responses[0].header.ty, FrameType::Error);
15617            let error = parse_error(&responses[0]);
15618            assert_eq!(error["code"], "version_unsupported");
15619            assert!(registry.get_module("aft").unwrap().is_none());
15620            assert_eq!(registry.active_registration_count().unwrap(), 0);
15621        }
15622    }
15623
15624    #[test]
15625    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15626        let registry = Arc::new(Registry::default());
15627        let forwarding = Arc::new(ForwardingTable::default());
15628        let handler =
15629            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15630        let module_connection = ConnectionId::new(301);
15631        let registration = registry
15632            .register_with_control_ops(
15633                manifest("aft-push", PROTOCOL_VERSION),
15634                PROTOCOL_VERSION,
15635                module_connection,
15636                module_baseline_control_ops(),
15637            )
15638            .unwrap();
15639        let (module_tx, _module_rx) = mpsc::channel(8);
15640        let endpoint = forwarding
15641            .register_module_connection(
15642                module_connection,
15643                "aft-push".to_string(),
15644                PROTOCOL_VERSION,
15645                manifest_concurrency(&registration.manifest),
15646                FrameSink::new(module_tx),
15647            )
15648            .unwrap();
15649
15650        // A push op this version does not know is ignored (forward-compat), not errored.
15651        let unknown = Frame::build(
15652            FrameType::Push,
15653            control_flags(),
15654            0,
15655            0,
15656            5,
15657            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15658        )
15659        .unwrap();
15660        let out = handler.handle_status_update(endpoint, unknown).unwrap();
15661        assert!(
15662            out.is_empty(),
15663            "unknown push op must be ignored, got {out:?}"
15664        );
15665
15666        // A malformed body for a KNOWN op is a real error worth surfacing.
15667        let malformed = Frame::build(
15668            FrameType::Push,
15669            control_flags(),
15670            0,
15671            0,
15672            6,
15673            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15674        )
15675        .unwrap();
15676        let out = handler.handle_status_update(endpoint, malformed).unwrap();
15677        assert_eq!(out.len(), 1);
15678        assert_eq!(out[0].header.ty, FrameType::Error);
15679        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15680    }
15681
15682    #[test]
15683    fn hello_rejected_when_connection_already_owns_client_routes() {
15684        let registry = Arc::new(Registry::default());
15685        let forwarding = Arc::new(ForwardingTable::default());
15686        let handler =
15687            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15688        // Commits a client route on connection 202 (bound to a module on conn 101).
15689        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
15690        let client_connection = ConnectionId::new(202);
15691
15692        // That same connection now tries to register as a module: rejected, so one
15693        // connection never holds both client-route and module-endpoint state.
15694        let responses = handler
15695            .handle_control(
15696                client_connection,
15697                hello_frame("aft-second", PROTOCOL_VERSION, 9),
15698            )
15699            .unwrap();
15700        assert_eq!(responses[0].header.ty, FrameType::Error);
15701        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15702        assert!(registry.get_module("aft-second").unwrap().is_none());
15703    }
15704
15705    #[tokio::test]
15706    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15707        let registry = Arc::new(Registry::default());
15708        let forwarding = Arc::new(ForwardingTable::default());
15709        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15710        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15711        hello_via_sink(
15712            &handler,
15713            &module_ctx,
15714            &mut module_rx,
15715            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15716        )
15717        .await;
15718        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15719        let pending = forwarding
15720            .begin_route_bind_relay_for_test(
15721                client_ctx.connection_id,
15722                client_ctx.egress.clone(),
15723                2,
15724                "alpha",
15725            )
15726            .unwrap();
15727        forwarding
15728            .complete_pending_relay(
15729                module_ctx.connection_id,
15730                pending.corr,
15731                RouteBindRelayOutcome::Accepted,
15732            )
15733            .unwrap();
15734        client_rx.try_recv().unwrap();
15735        for module_id in ["beta", "alpha"] {
15736            let replies = handler
15737                .handle_control_frame(
15738                    &module_ctx,
15739                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15740                )
15741                .await
15742                .unwrap();
15743            assert_eq!(replies.len(), 1, "second HELLO must be refused");
15744            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15745        }
15746        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15747        assert!(registry.get_module("beta").unwrap().is_none());
15748        assert!(matches!(
15749            forwarding
15750                .lookup_data_route(
15751                    client_ctx.connection_id,
15752                    pending.client_channel,
15753                    pending.client_epoch,
15754                )
15755                .unwrap(),
15756            DataRoute::Client(DataRouteState::Bound(_))
15757        ));
15758        assert!(handler
15759            .hello_launch_nonces
15760            .lock()
15761            .unwrap()
15762            .presented(module_ctx.connection_id, Some("alpha-nonce")));
15763        assert!(module_rx.try_recv().is_err());
15764    }
15765
15766    #[test]
15767    fn reserved_module_hello_requires_matching_launch_nonce() {
15768        let registry = Arc::new(Registry::default());
15769        let supervisor = SupervisorHandle::new();
15770        // The supervisor recorded the nonce it injected when it spawned the reserved
15771        // module; the HELLO verifier checks against the same shared handle.
15772        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15773        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15774
15775        // A HELLO with NO nonce is rejected.
15776        let no_nonce = handler
15777            .handle_control(
15778                ConnectionId::new(1),
15779                hello_frame("vault", PROTOCOL_VERSION, 1),
15780            )
15781            .unwrap();
15782        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15783        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15784        assert!(registry.get_module("vault").unwrap().is_none());
15785
15786        // A HELLO with the WRONG nonce is rejected.
15787        let wrong = handler
15788            .handle_control(
15789                ConnectionId::new(2),
15790                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15791            )
15792            .unwrap();
15793        assert_eq!(wrong[0].header.ty, FrameType::Error);
15794        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15795        assert!(registry.get_module("vault").unwrap().is_none());
15796
15797        // A HELLO with the CORRECT nonce registers.
15798        let ok = handler
15799            .handle_control(
15800                ConnectionId::new(3),
15801                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15802            )
15803            .unwrap();
15804        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15805        assert!(registry.get_module("vault").unwrap().is_some());
15806    }
15807
15808    #[test]
15809    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15810        let registry = Arc::new(Registry::default());
15811        let supervisor = SupervisorHandle::new();
15812        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15813        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15814        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15815
15816        let squat = handler
15817            .handle_control(
15818                ConnectionId::new(1),
15819                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15820            )
15821            .unwrap();
15822        assert_eq!(squat[0].header.ty, FrameType::Error);
15823        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15824        assert!(parse_error(&squat[0])["message"]
15825            .as_str()
15826            .unwrap()
15827            .contains("fed:"));
15828
15829        let accepted_peer = handler
15830            .handle_control(
15831                ConnectionId::new(2),
15832                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15833            )
15834            .unwrap();
15835        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15836
15837        let accepted_short = handler
15838            .handle_control(
15839                ConnectionId::new(3),
15840                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15841            )
15842            .unwrap();
15843        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15844
15845        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15846            let response = handler
15847                .handle_control(
15848                    ConnectionId::new(conn),
15849                    hello_frame(module_id, PROTOCOL_VERSION, conn),
15850                )
15851                .unwrap();
15852            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15853        }
15854    }
15855
15856    #[test]
15857    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15858        let registry = Arc::new(Registry::default());
15859        let supervisor = SupervisorHandle::new();
15860        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15861        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15862        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15863        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15864
15865        let owner_nonce = handler
15866            .handle_control(
15867                ConnectionId::new(1),
15868                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15869            )
15870            .unwrap();
15871        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15872        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15873        assert!(registry.get_module("fed:special").unwrap().is_none());
15874
15875        let exact_nonce = handler
15876            .handle_control(
15877                ConnectionId::new(2),
15878                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15879            )
15880            .unwrap();
15881        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15882        assert!(registry.get_module("fed:special").unwrap().is_some());
15883    }
15884
15885    #[test]
15886    fn non_reserved_module_ignores_launch_nonce() {
15887        let registry = Arc::new(Registry::default());
15888        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
15889        // registration succeeds whether a spawned process echoes a nonce or not.
15890        let handler = ControlHandler::new(Arc::clone(&registry));
15891        let no_nonce = handler
15892            .handle_control(
15893                ConnectionId::new(1),
15894                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15895            )
15896            .unwrap();
15897        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15898        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15899
15900        let echoed_nonce = handler
15901            .handle_control(
15902                ConnectionId::new(2),
15903                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15904            )
15905            .unwrap();
15906        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15907        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15908    }
15909
15910    #[test]
15911    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15912        let handler = ControlHandler::default();
15913        let conn = ConnectionId::new(1);
15914        let malformed = Frame::build(
15915            FrameType::Hello,
15916            control_flags(),
15917            0,
15918            0,
15919            3,
15920            b"{not json".to_vec(),
15921        )
15922        .unwrap();
15923
15924        let error = handler.handle_control(conn, malformed).unwrap();
15925        assert_eq!(error[0].header.ty, FrameType::Error);
15926        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15927
15928        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15929        let pong = handler.handle_control(conn, ping).unwrap();
15930        assert_eq!(pong[0].header.ty, FrameType::Pong);
15931        assert_eq!(pong[0].header.corr, 4);
15932    }
15933
15934    #[test]
15935    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15936        let registry = Arc::new(Registry::default());
15937        let handler = ControlHandler::new(Arc::clone(&registry));
15938
15939        handler
15940            .handle_control(
15941                ConnectionId::new(1),
15942                hello_frame("aft", PROTOCOL_VERSION, 1),
15943            )
15944            .unwrap();
15945        let duplicate = handler
15946            .handle_control(
15947                ConnectionId::new(2),
15948                hello_frame("aft", PROTOCOL_VERSION, 2),
15949            )
15950            .unwrap();
15951
15952        assert_eq!(duplicate[0].header.ty, FrameType::Error);
15953        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15954        let registration = registry.get_module("aft").unwrap().unwrap();
15955        assert_eq!(registration.connection_id, ConnectionId::new(1));
15956    }
15957
15958    #[test]
15959    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15960        let registry = Arc::new(Registry::default());
15961        let forwarding = Arc::new(ForwardingTable::default());
15962        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15963        let handler =
15964            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15965                .with_process_liveness(process_liveness);
15966        let (ctx, route_channel, route_epoch) =
15967            bind_liveness_route(&registry, &forwarding, "aft-dead");
15968        let responses = handler
15969            .handle_route_poll(
15970                &ctx,
15971                route_poll_frame(41, PollKind::Liveness, route_channel),
15972                route_channel,
15973                route_epoch,
15974                PollKind::Liveness,
15975            )
15976            .unwrap();
15977
15978        assert_eq!(responses.len(), 1);
15979        assert_eq!(responses[0].header.ty, FrameType::Response);
15980        assert_route_poll_liveness(&responses[0], false);
15981    }
15982
15983    #[test]
15984    fn liveness_poll_without_process_source_uses_bound_route() {
15985        let registry = Arc::new(Registry::default());
15986        let forwarding = Arc::new(ForwardingTable::default());
15987        let handler =
15988            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15989        let (ctx, route_channel, route_epoch) =
15990            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
15991        let responses = handler
15992            .handle_route_poll(
15993                &ctx,
15994                route_poll_frame(42, PollKind::Liveness, route_channel),
15995                route_channel,
15996                route_epoch,
15997                PollKind::Liveness,
15998            )
15999            .unwrap();
16000
16001        assert_route_poll_liveness(&responses[0], true);
16002    }
16003
16004    #[test]
16005    fn liveness_poll_untracked_process_source_uses_bound_route() {
16006        let registry = Arc::new(Registry::default());
16007        let forwarding = Arc::new(ForwardingTable::default());
16008        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
16009        let handler =
16010            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16011                .with_process_liveness(process_liveness);
16012        let (ctx, route_channel, route_epoch) =
16013            bind_liveness_route(&registry, &forwarding, "aft-untracked");
16014        let responses = handler
16015            .handle_route_poll(
16016                &ctx,
16017                route_poll_frame(43, PollKind::Liveness, route_channel),
16018                route_channel,
16019                route_epoch,
16020                PollKind::Liveness,
16021            )
16022            .unwrap();
16023
16024        assert_route_poll_liveness(&responses[0], true);
16025    }
16026
16027    #[tokio::test]
16028    async fn unknown_op_returns_unknown_control_op() {
16029        let handler = ControlHandler::default();
16030        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
16031        let request = Frame::build(
16032            FrameType::Request,
16033            control_flags(),
16034            0,
16035            0,
16036            55,
16037            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
16038        )
16039        .unwrap();
16040
16041        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16042
16043        assert_eq!(response.len(), 1);
16044        assert_eq!(response[0].header.ty, FrameType::Error);
16045        assert_eq!(response[0].header.corr, 55);
16046        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
16047    }
16048
16049    #[tokio::test]
16050    async fn supervisor_provenance_rejects_unknown_exact_module() {
16051        let handler = ControlHandler::default();
16052        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
16053        let request = Frame::build(
16054            FrameType::Request,
16055            control_flags(),
16056            0,
16057            0,
16058            57,
16059            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
16060        )
16061        .unwrap();
16062
16063        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16064
16065        assert_eq!(response.len(), 1);
16066        assert_eq!(response[0].header.ty, FrameType::Error);
16067        assert_eq!(response[0].header.corr, 57);
16068        let error = parse_error(&response[0]);
16069        assert_eq!(error["code"], "unknown_module");
16070        assert_eq!(error["message"], "module_id 'missing' is not supervised");
16071    }
16072
16073    #[test]
16074    fn provenance_probe_override_keeps_handler_tests_deterministic() {
16075        let expected = subc_control::RunningImageAgreement::Unavailable {
16076            reason: subc_control::RunningImageUnavailableReason::HashFailed,
16077        };
16078        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
16079        assert_eq!(handler.provenance_probe_override, Some(expected));
16080    }
16081
16082    #[test]
16083    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
16084        let verdict = reload_verdict(
16085            std::path::Path::new("/bin/new"),
16086            Some(std::path::Path::new("/bin/old")),
16087            subc_control::RunningImageAgreement::Unavailable {
16088                reason: subc_control::RunningImageUnavailableReason::HashFailed,
16089            },
16090        );
16091        assert!(matches!(
16092            verdict.path,
16093            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
16094                if configured == std::path::Path::new("/bin/new")
16095                    && spawned_from == std::path::Path::new("/bin/old")
16096        ));
16097    }
16098
16099    #[test]
16100    fn launch_spec_without_running_digest_is_unavailable_not_match() {
16101        use subc_control::{
16102            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
16103        };
16104        let configured = crate::supervise::ModuleSpec {
16105            carry_containment: false,
16106            module_id: "legacy".into(),
16107            program: "/module".into(),
16108            args: Vec::new(),
16109            env: Vec::new(),
16110            reserved: false,
16111            reserved_prefixes: Vec::new(),
16112            protocol: subc_control::ModuleProtocol::Subc,
16113            overlap: Default::default(),
16114        };
16115        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
16116        assert_eq!(
16117            launch_spec_agreement(&configured, false, Some(&digest)),
16118            Agreement::Unavailable {
16119                reason: Reason::NotRunning
16120            }
16121        );
16122        assert_eq!(
16123            launch_spec_agreement(&configured, true, None),
16124            Agreement::Unavailable {
16125                reason: Reason::SpawnedSpecUnavailable
16126            }
16127        );
16128        assert_eq!(
16129            launch_spec_agreement(&configured, true, Some(&digest)),
16130            Agreement::Match
16131        );
16132    }
16133
16134    #[cfg(windows)]
16135    #[test]
16136    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16137        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16138        let configured = dir.join("module.exe");
16139        std::fs::write(&configured, b"executable path fixture").unwrap();
16140        let spawned = std::fs::canonicalize(&configured).unwrap();
16141        assert_ne!(
16142            configured, spawned,
16143            "fixture must exercise different path spellings"
16144        );
16145        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16146        let lower_case = configured.to_string_lossy().to_lowercase();
16147        for spelling in [
16148            &configured,
16149            Path::new(&forward_slashes),
16150            Path::new(&lower_case),
16151        ] {
16152            let verdict = reload_verdict(
16153                spelling,
16154                Some(&spawned),
16155                subc_control::RunningImageAgreement::Unavailable {
16156                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16157                },
16158            );
16159            assert_eq!(
16160                verdict.path,
16161                ReloadPathAgreement::Match,
16162                "configured {spelling:?}, spawned {spawned:?}"
16163            );
16164        }
16165    }
16166
16167    #[cfg(windows)]
16168    #[test]
16169    fn windows_reload_path_keeps_different_existing_program_pending() {
16170        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16171        let old = dir.join("old.exe");
16172        let new = dir.join("new.exe");
16173        std::fs::write(&old, b"same executable content").unwrap();
16174        std::fs::copy(&old, &new).unwrap();
16175        let spawned = std::fs::canonicalize(&old).unwrap();
16176        let image = subc_control::RunningImageAgreement::Unavailable {
16177            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16178        };
16179        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16180        assert!(
16181            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16182            if configured == new && spawned_from == spawned)
16183        );
16184        let missing = dir.join("missing.exe");
16185        assert!(matches!(
16186            reload_verdict(&missing, Some(&spawned), image).path,
16187            ReloadPathAgreement::Mismatch { .. }
16188        ));
16189    }
16190
16191    #[test]
16192    fn reload_verdict_detects_replaced_image_at_same_path() {
16193        let image = subc_control::RunningImageAgreement::Mismatch {
16194            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16195                digest: "old".into(),
16196            },
16197            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16198                digest: "new".into(),
16199            },
16200        };
16201        let verdict = reload_verdict(
16202            std::path::Path::new("/bin/same"),
16203            Some(std::path::Path::new("/bin/same")),
16204            image.clone(),
16205        );
16206        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16207        assert_eq!(verdict.image, image);
16208    }
16209
16210    #[test]
16211    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16212        let image = subc_control::RunningImageAgreement::Unavailable {
16213            reason: subc_control::RunningImageUnavailableReason::NotRunning,
16214        };
16215        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16216        assert_eq!(
16217            verdict.path,
16218            subc_control::ReloadPathAgreement::Unavailable {
16219                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16220            }
16221        );
16222        assert_eq!(verdict.image, image);
16223
16224        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16225            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16226        };
16227        let verdict = reload_verdict(
16228            std::path::Path::new("/bin/same"),
16229            Some(std::path::Path::new("/bin/same")),
16230            unconfirmed.clone(),
16231        );
16232        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16233        assert_eq!(verdict.image, unconfirmed);
16234    }
16235
16236    #[test]
16237    fn reload_verdict_preserves_each_image_unavailability_reason() {
16238        use subc_control::RunningImageUnavailableReason as Reason;
16239
16240        for reason in [
16241            Reason::NotRunning,
16242            Reason::UnsupportedPlatform,
16243            Reason::RunningExecutableUnreadable,
16244            Reason::SpawnedPathUnreadable,
16245            Reason::HashFailed,
16246            Reason::ProcessIdentityUnconfirmed,
16247            Reason::Unknown("future_probe_reason".to_string()),
16248        ] {
16249            let image = subc_control::RunningImageAgreement::Unavailable {
16250                reason: reason.clone(),
16251            };
16252            let verdict = reload_verdict(
16253                std::path::Path::new("/bin/same"),
16254                Some(std::path::Path::new("/bin/same")),
16255                image.clone(),
16256            );
16257            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16258            assert_eq!(verdict.image, image, "{reason:?}");
16259        }
16260    }
16261
16262    #[tokio::test]
16263    async fn malformed_control_bodies_return_invalid_control_body() {
16264        let handler = ControlHandler::default();
16265        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16266
16267        for (corr, body) in [
16268            (56, br#"{"route_channel":1}"#.as_slice()),
16269            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16270            (
16271                58,
16272                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16273            ),
16274        ] {
16275            let request = Frame::build(
16276                FrameType::Request,
16277                control_flags(),
16278                0,
16279                0,
16280                corr,
16281                body.to_vec(),
16282            )
16283            .unwrap();
16284            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16285
16286            assert_eq!(response.len(), 1);
16287            assert_eq!(response[0].header.ty, FrameType::Error);
16288            assert_eq!(response[0].header.corr, corr);
16289            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16290        }
16291    }
16292
16293    #[tokio::test]
16294    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16295        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16296        let registry = Arc::new(Registry::default());
16297        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16298        let router = Router::with_control_handler(Arc::clone(&control));
16299        let connection = router.begin_connection();
16300        let (ctx, mut rx) = route_ctx(connection.id());
16301
16302        router
16303            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16304            .await
16305            .unwrap();
16306        let response = rx.recv().await.unwrap();
16307        let ack = parse_ack(&response);
16308        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16309        let channel = 1;
16310
16311        let goodbye =
16312            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16313        router.route_for_connection(&ctx, goodbye).await.unwrap();
16314        assert!(rx.try_recv().is_err());
16315        assert!(registry.get_module("aft").unwrap().is_none());
16316
16317        router
16318            .route_for_connection(&ctx, channel_request(channel, 13))
16319            .await
16320            .unwrap();
16321        let error_frame = rx.recv().await.unwrap();
16322        assert_eq!(error_frame.header.ty, FrameType::Error);
16323        assert_eq!(error_frame.header.channel, channel);
16324        let captured = crate::router::test_log::captured_logs(&logs);
16325        assert_eq!(
16326            captured
16327                .lines()
16328                .filter(|line| {
16329                    line.contains("module_id=aft")
16330                        && line.contains("reason=explicit_goodbye")
16331                        && line.contains("module registration ended")
16332                })
16333                .count(),
16334            1,
16335            "unexpected GOODBYE registry log: {captured}"
16336        );
16337    }
16338
16339    #[tokio::test]
16340    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16341        let registry = Arc::new(Registry::default());
16342        let handler = ControlHandler::new(registry).with_capability_config(
16343            [("prov".to_string(), true), ("cons".to_string(), true)],
16344            BTreeMap::new(),
16345        );
16346        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16347        register_capability_manifest(
16348            &handler,
16349            &provider_ctx,
16350            &mut provider_rx,
16351            capability_manifest("prov", &["thing/v1"], &[]),
16352            1,
16353        )
16354        .await;
16355        let mut consumer = capability_manifest("cons", &[], &[]);
16356        consumer.capabilities.as_mut().unwrap().requires.push(
16357            subc_protocol::manifest::CapabilityRequirement {
16358                capability: "thing/v1".to_string(),
16359                need: subc_protocol::manifest::CapabilityNeed::Required,
16360            },
16361        );
16362        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16363        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16364        assert_eq!(
16365            handler.capability_evaluator.verdict("cons", "thing/v1"),
16366            Some(CapabilityVerdict::Provided)
16367        );
16368        let (mut client_rx, _) = open_route_for_capability_test(
16369            &handler,
16370            &provider_ctx,
16371            &mut provider_rx,
16372            703,
16373            3,
16374            "prov",
16375            None,
16376        )
16377        .await;
16378        let goodbye =
16379            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16380        handler
16381            .handle_control_frame(&provider_ctx, goodbye)
16382            .await
16383            .unwrap();
16384        assert_eq!(
16385            handler.capability_evaluator.verdict("cons", "thing/v1"),
16386            Some(CapabilityVerdict::NeverProvided)
16387        );
16388        let closed = client_rx
16389            .try_recv()
16390            .expect("GOODBYE pushes route.closed before route GOODBYE");
16391        assert!(
16392            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16393            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16394        );
16395        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16396        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16397    }
16398
16399    struct RouteClosedFixture {
16400        handler: ControlHandler,
16401        #[cfg(unix)]
16402        supervisor: Supervisor,
16403        module_ctx: RouteCtx,
16404        #[cfg(unix)]
16405        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16406        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16407        route: Arc<crate::forwarding::RouteBinding>,
16408    }
16409
16410    async fn route_closed_fixture() -> RouteClosedFixture {
16411        let registry = Arc::new(Registry::default());
16412        let handler = ControlHandler::new(Arc::clone(&registry));
16413        #[cfg(unix)]
16414        let supervisor = Supervisor::new(registry, RestartPolicy::default())
16415            .with_forwarding(Arc::clone(&handler.forwarding));
16416        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16417        register_capability_manifest(
16418            &handler,
16419            &module_ctx,
16420            &mut module_rx,
16421            capability_manifest("prov", &[], &[]),
16422            1,
16423        )
16424        .await;
16425        let (client_rx, bind) = open_route_for_capability_test(
16426            &handler,
16427            &module_ctx,
16428            &mut module_rx,
16429            702,
16430            2,
16431            "prov",
16432            None,
16433        )
16434        .await;
16435        let ModuleControlRequest::RouteBind {
16436            route_channel: channel,
16437            epoch: route_epoch,
16438            ..
16439        } = bind
16440        else {
16441            panic!("expected route.bind");
16442        };
16443        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16444            handler
16445                .forwarding
16446                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16447                .unwrap()
16448        else {
16449            panic!("the acknowledged route must be bound");
16450        };
16451        RouteClosedFixture {
16452            handler,
16453            #[cfg(unix)]
16454            supervisor,
16455            module_ctx,
16456            #[cfg(unix)]
16457            module_rx,
16458            client_rx,
16459            route,
16460        }
16461    }
16462
16463    impl RouteClosedFixture {
16464        #[cfg(unix)]
16465        async fn cut(&mut self) {
16466            let notices = async {
16467                let mut draining = self.module_rx.recv().await.unwrap();
16468                assert!(matches!(
16469                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16470                        &draining.body
16471                    )
16472                    .unwrap(),
16473                    subc_protocol::session::ModuleControlCommand::Draining {
16474                        reason: RouteCloseReason::Disable,
16475                        ..
16476                    }
16477                ));
16478                draining.flushed.take().unwrap().send(()).unwrap();
16479                let mut closing = self.client_rx.recv().await.unwrap();
16480                assert!(matches!(
16481                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16482                    ClientControlPush::RouteClosing {
16483                        reason: RouteCloseReason::Restart,
16484                        ..
16485                    }
16486                ));
16487                closing.flushed.take().unwrap().send(()).unwrap();
16488            };
16489            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16490            result.unwrap();
16491        }
16492
16493        fn disconnect(&mut self) -> ClientControlPush {
16494            self.handler
16495                .cleanup_connection(self.module_ctx.connection_id)
16496                .unwrap();
16497            let closed = self.client_rx.try_recv().unwrap();
16498            assert_eq!(closed.header.ty, FrameType::Push);
16499            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16500            assert!(matches!(
16501                &push,
16502                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16503                    if channels == &[self.route.client_channel]
16504            ));
16505            assert_eq!(
16506                self.client_rx.try_recv().unwrap().header.ty,
16507                FrameType::Goodbye
16508            );
16509            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16510            push
16511        }
16512    }
16513
16514    #[cfg(unix)]
16515    #[tokio::test(start_paused = true)]
16516    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16517        let mut fixture = route_closed_fixture().await;
16518        fixture.cut().await;
16519        assert!(matches!(
16520            fixture.disconnect(),
16521            ClientControlPush::RouteClosed {
16522                reason: RouteCloseReason::Restart,
16523                drained: true,
16524                ..
16525            }
16526        ));
16527    }
16528
16529    #[cfg(unix)]
16530    #[tokio::test(start_paused = true)]
16531    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16532        let mut fixture = route_closed_fixture().await;
16533        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16534        let started = tokio::time::Instant::now();
16535        fixture.cut().await;
16536        assert!(started.elapsed() >= Duration::from_secs(2));
16537        assert_eq!(fixture.route.flow.in_flight(), 1);
16538        assert!(matches!(
16539            fixture.disconnect(),
16540            ClientControlPush::RouteClosed {
16541                reason: RouteCloseReason::Restart,
16542                drained: false,
16543                ..
16544            }
16545        ));
16546    }
16547
16548    #[tokio::test]
16549    async fn module_connection_crash_reports_crash_and_not_drained() {
16550        let mut fixture = route_closed_fixture().await;
16551        // The server calls cleanup_connection on module EOF. Without a planned
16552        // daemon shutdown, that disconnect must remain a crash with no prior
16553        // route.closing notice, even when the route has no in-flight calls.
16554        assert!(matches!(
16555            fixture.disconnect(),
16556            ClientControlPush::RouteClosed {
16557                reason: RouteCloseReason::Crash,
16558                drained: false,
16559                ..
16560            }
16561        ));
16562    }
16563
16564    #[tokio::test]
16565    async fn dropping_router_connection_releases_registration() {
16566        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16567        let registry = Arc::new(Registry::default());
16568        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16569        let router = Router::with_control_handler(Arc::clone(&control));
16570        let connection = router.begin_connection();
16571        let connection_id = connection.id();
16572        let (ctx, mut rx) = route_ctx(connection_id);
16573
16574        router
16575            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16576            .await
16577            .unwrap();
16578        let response = rx.recv().await.unwrap();
16579        let ack = parse_ack(&response);
16580        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16581        assert!(registry.get_module("aft").unwrap().is_some());
16582
16583        drop(connection);
16584
16585        assert!(registry.get_module("aft").unwrap().is_none());
16586        assert_eq!(registry.active_registration_count().unwrap(), 0);
16587
16588        control
16589            .cleanup_connection(ConnectionId::new(u64::MAX))
16590            .unwrap();
16591        let captured = crate::router::test_log::captured_logs(&logs);
16592        println!("captured registry lifecycle logs:\n{captured}");
16593        let events: Vec<_> = captured
16594            .lines()
16595            .filter(|line| {
16596                line.contains("module registered module_id=aft ")
16597                    || line.contains("module registration ended")
16598            })
16599            .collect();
16600        assert_eq!(
16601            events.len(),
16602            2,
16603            "unexpected registry lifecycle logs: {captured}"
16604        );
16605        assert!(events[0].contains("module registered module_id=aft "));
16606        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16607        assert!(events[1].contains(&format!(
16608            "module_id=aft connection_id={}",
16609            connection_id.get()
16610        )));
16611        assert!(events[1].contains("reason=connection_closed"));
16612        assert!(events[1].contains("module registration ended"));
16613        assert_eq!(
16614            captured
16615                .lines()
16616                .filter(|line| line.contains("module registered module_id=aft "))
16617                .count(),
16618            1,
16619            "legacy registration admission line must appear once: {captured}"
16620        );
16621    }
16622
16623    #[tokio::test]
16624    async fn hello_registration_keeps_legacy_module_registered_line_once() {
16625        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16626        let registry = Arc::new(Registry::default());
16627        let control = ControlHandler::new(Arc::clone(&registry));
16628        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16629        hello_via_sink(
16630            &control,
16631            &ctx,
16632            &mut rx,
16633            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16634        )
16635        .await;
16636
16637        let captured = crate::router::test_log::captured_logs(&logs);
16638        let admissions: Vec<_> = captured
16639            .lines()
16640            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16641            .collect();
16642        assert_eq!(
16643            admissions.len(),
16644            1,
16645            "legacy admission line must remain exactly once: {captured}"
16646        );
16647        assert!(admissions[0].contains("routable_provider=true"));
16648        assert!(admissions[0].contains("connection_id=777"));
16649    }
16650
16651    fn capability_manifest(
16652        module_id: &str,
16653        provides: &[&str],
16654        must_never_reach: &[&str],
16655    ) -> ModuleManifest {
16656        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16657        manifest.capabilities = Some(CapabilityDeclarations {
16658            provides: provides
16659                .iter()
16660                .map(|capability| (*capability).to_string())
16661                .collect(),
16662            requires: Vec::new(),
16663            must_never_reach: must_never_reach
16664                .iter()
16665                .map(|capability| (*capability).to_string())
16666                .collect(),
16667        });
16668        manifest
16669    }
16670
16671    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16672        Frame::build(
16673            FrameType::Hello,
16674            control_flags(),
16675            0,
16676            0,
16677            corr,
16678            serde_json::to_vec(&ModuleHelloBody {
16679                protocol_ver: manifest.protocol_ver,
16680                manifest,
16681                control_ops: None,
16682                launch_nonce: None,
16683            })
16684            .expect("capability test HELLO serializes"),
16685        )
16686        .expect("capability test HELLO frame builds")
16687    }
16688
16689    fn catalog_update_with_capabilities_frame(
16690        corr: u64,
16691        capabilities: CapabilityDeclarations,
16692    ) -> Frame {
16693        Frame::build(
16694            FrameType::Request,
16695            control_flags(),
16696            0,
16697            0,
16698            corr,
16699            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16700                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16701                capabilities: Some(capabilities),
16702                ready: None,
16703            })
16704            .expect("capability catalog.update serializes"),
16705        )
16706        .expect("capability catalog.update frame builds")
16707    }
16708
16709    async fn register_capability_manifest(
16710        handler: &ControlHandler,
16711        ctx: &RouteCtx,
16712        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16713        manifest: ModuleManifest,
16714        corr: u64,
16715    ) {
16716        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16717    }
16718
16719    async fn open_route_for_capability_test(
16720        handler: &ControlHandler,
16721        target_ctx: &RouteCtx,
16722        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16723        client_connection_id: u64,
16724        corr: u64,
16725        target_module_id: &str,
16726        consumer_identity: Option<ConsumerIdentity>,
16727    ) -> (
16728        mpsc::Receiver<crate::router::OutboundFrame>,
16729        ModuleControlRequest,
16730    ) {
16731        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16732        let route_handler = handler.clone();
16733        let target_module_id = target_module_id.to_string();
16734        let route_task = tokio::spawn(async move {
16735            route_handler
16736                .handle_control_frame(
16737                    &client_ctx,
16738                    route_open_frame_with_admission_facts(
16739                        corr,
16740                        &target_module_id,
16741                        unique_project_root("admission-facts"),
16742                        consumer_identity,
16743                        None,
16744                    ),
16745                )
16746                .await
16747                .expect("capability test route.open succeeds")
16748        });
16749        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16750            .await
16751            .expect("capability test route.open must reach route.bind")
16752            .expect("target control receiver stays open");
16753        let bind_request: ModuleControlRequest =
16754            serde_json::from_slice(&bind.body).expect("route.bind decodes");
16755        handler
16756            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16757            .await
16758            .expect("capability test route.bind ACK succeeds");
16759        assert!(route_task.await.expect("route.open task joins").is_empty());
16760        let opened = client_rx
16761            .recv()
16762            .await
16763            .expect("successful route.open publishes a response");
16764        assert!(matches!(
16765            serde_json::from_slice::<ClientControlResponse>(&opened.body),
16766            Ok(ClientControlResponse::RouteOpen { .. })
16767        ));
16768        (client_rx, bind_request)
16769    }
16770
16771    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16772        assert_eq!(frame.header.ty, FrameType::Push);
16773        assert_eq!(frame.header.channel, 0);
16774        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16775            .expect("route.closed control push decodes");
16776        let ClientControlPush::RouteClosed { channels, .. } = &push else {
16777            panic!("expected route.closed");
16778        };
16779        assert_eq!(channels.len(), 1, "exactly one violating route closed");
16780        let closed_channel = channels[0];
16781        let channels = channels.clone();
16782        assert_eq!(
16783            push,
16784            ClientControlPush::RouteClosed {
16785                module_id: target_module_id.to_string(),
16786                channels,
16787                reason: RouteCloseReason::CapabilityDenied,
16788                drained: false,
16789                abandoned: 0,
16790                excluded_subscriptions: 0,
16791                terminal: Some(false),
16792            }
16793        );
16794        closed_channel
16795    }
16796
16797    #[tokio::test]
16798    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16799        let registry = Arc::new(Registry::default());
16800        let forwarding = Arc::new(ForwardingTable::default());
16801        let supervisor = SupervisorHandle::new();
16802        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16803        let handler =
16804            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16805                .with_supervisor(supervisor);
16806        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16807        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16808        register_capability_manifest(
16809            &handler,
16810            &target_ctx,
16811            &mut target_rx,
16812            capability_manifest("target", &["credentials-provider/v1"], &[]),
16813            1,
16814        )
16815        .await;
16816        register_capability_manifest(
16817            &handler,
16818            &opener_ctx,
16819            &mut opener_rx,
16820            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16821            2,
16822        )
16823        .await;
16824
16825        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16826        let replies = handler
16827            .handle_control_frame(
16828                &client_ctx,
16829                route_open_frame_with_admission_facts(
16830                    3,
16831                    "target",
16832                    unique_project_root("admission-facts"),
16833                    Some(ConsumerIdentity {
16834                        module_id: "opener".to_string(),
16835                        launch_nonce: "opener-nonce".to_string(),
16836                    }),
16837                    None,
16838                ),
16839            )
16840            .await
16841            .expect("denied route.open returns a typed frame");
16842        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16843        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16844        assert!(
16845            target_rx.try_recv().is_err(),
16846            "forbidden route.open must not relay route.bind"
16847        );
16848    }
16849
16850    #[tokio::test]
16851    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16852        let registry = Arc::new(Registry::default());
16853        let forwarding = Arc::new(ForwardingTable::default());
16854        let supervisor = SupervisorHandle::new();
16855        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16856        let handler =
16857            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16858                .with_supervisor(supervisor);
16859        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16860        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16861        register_capability_manifest(
16862            &handler,
16863            &target_ctx,
16864            &mut target_rx,
16865            capability_manifest("target", &["credentials-provider/v1"], &[]),
16866            1,
16867        )
16868        .await;
16869        register_capability_manifest(
16870            &handler,
16871            &old_opener_ctx,
16872            &mut old_opener_rx,
16873            capability_manifest("opener", &[], &[]),
16874            2,
16875        )
16876        .await;
16877        let (mut client_rx, _) = open_route_for_capability_test(
16878            &handler,
16879            &target_ctx,
16880            &mut target_rx,
16881            712,
16882            3,
16883            "target",
16884            Some(ConsumerIdentity {
16885                module_id: "opener".to_string(),
16886                launch_nonce: "opener-nonce".to_string(),
16887            }),
16888        )
16889        .await;
16890        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16891
16892        handler
16893            .cleanup_connection(old_opener_ctx.connection_id)
16894            .expect("old opener registration cleans up");
16895        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16896        register_capability_manifest(
16897            &handler,
16898            &new_opener_ctx,
16899            &mut new_opener_rx,
16900            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16901            4,
16902        )
16903        .await;
16904
16905        let closed_channel = assert_capability_denied_push(
16906            client_rx
16907                .try_recv()
16908                .expect("HELLO deny addition must emit route.closed")
16909                .frame,
16910            "target",
16911        );
16912        let goodbye = client_rx
16913            .try_recv()
16914            .expect("HELLO deny addition must send the client a route GOODBYE")
16915            .frame;
16916        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16917        assert_eq!(goodbye.header.channel, closed_channel);
16918        assert_ne!(goodbye.header.channel, 0);
16919        assert_ne!(goodbye.header.epoch, 0);
16920        assert!(client_rx.try_recv().is_err());
16921        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16922        assert!(matches!(
16923            target_rx.try_recv(),
16924            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16925        ));
16926    }
16927
16928    #[tokio::test]
16929    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16930        let registry = Arc::new(Registry::default());
16931        let forwarding = Arc::new(ForwardingTable::default());
16932        let supervisor = SupervisorHandle::new();
16933        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16934        let handler =
16935            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16936                .with_supervisor(supervisor);
16937        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16938        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16939        register_capability_manifest(
16940            &handler,
16941            &target_ctx,
16942            &mut target_rx,
16943            capability_manifest("target", &[], &[]),
16944            1,
16945        )
16946        .await;
16947        register_capability_manifest(
16948            &handler,
16949            &opener_ctx,
16950            &mut opener_rx,
16951            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16952            2,
16953        )
16954        .await;
16955        let (mut client_rx, _) = open_route_for_capability_test(
16956            &handler,
16957            &target_ctx,
16958            &mut target_rx,
16959            722,
16960            3,
16961            "target",
16962            Some(ConsumerIdentity {
16963                module_id: "opener".to_string(),
16964                launch_nonce: "opener-nonce".to_string(),
16965            }),
16966        )
16967        .await;
16968        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16969
16970        let replies = handler
16971            .handle_control_frame(
16972                &target_ctx,
16973                catalog_update_with_capabilities_frame(
16974                    4,
16975                    CapabilityDeclarations {
16976                        provides: vec!["credentials-provider/v1".to_string()],
16977                        requires: Vec::new(),
16978                        must_never_reach: Vec::new(),
16979                    },
16980                ),
16981            )
16982            .await
16983            .expect("claim catalog.update succeeds");
16984        assert!(matches!(
16985            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16986            Ok(ModuleControlResponseToModule::CatalogUpdate {})
16987        ));
16988        let closed_channel = assert_capability_denied_push(
16989            client_rx
16990                .try_recv()
16991                .expect("claim addition must emit route.closed")
16992                .frame,
16993            "target",
16994        );
16995        let goodbye = client_rx
16996            .try_recv()
16997            .expect("claim addition must send the client a route GOODBYE")
16998            .frame;
16999        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17000        assert_eq!(goodbye.header.channel, closed_channel);
17001        assert_ne!(goodbye.header.channel, 0);
17002        assert_ne!(goodbye.header.epoch, 0);
17003        assert!(client_rx.try_recv().is_err());
17004        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17005        assert!(matches!(
17006            target_rx.try_recv(),
17007            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17008        ));
17009    }
17010
17011    #[tokio::test]
17012    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
17013        let registry = Arc::new(Registry::default());
17014        let forwarding = Arc::new(ForwardingTable::default());
17015        let supervisor = SupervisorHandle::new();
17016        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17017        let handler =
17018            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17019                .with_supervisor(supervisor);
17020        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
17021        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
17022        register_capability_manifest(
17023            &handler,
17024            &target_ctx,
17025            &mut target_rx,
17026            capability_manifest("target", &["credentials-provider/v1"], &[]),
17027            1,
17028        )
17029        .await;
17030        register_capability_manifest(
17031            &handler,
17032            &opener_ctx,
17033            &mut opener_rx,
17034            capability_manifest("opener", &[], &[]),
17035            2,
17036        )
17037        .await;
17038        let (mut client_rx, _) = open_route_for_capability_test(
17039            &handler,
17040            &target_ctx,
17041            &mut target_rx,
17042            732,
17043            3,
17044            "target",
17045            Some(ConsumerIdentity {
17046                module_id: "opener".to_string(),
17047                launch_nonce: "opener-nonce".to_string(),
17048            }),
17049        )
17050        .await;
17051        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17052
17053        handler
17054            .handle_control_frame(
17055                &target_ctx,
17056                catalog_update_with_capabilities_frame(
17057                    4,
17058                    CapabilityDeclarations {
17059                        provides: Vec::new(),
17060                        requires: Vec::new(),
17061                        must_never_reach: Vec::new(),
17062                    },
17063                ),
17064            )
17065            .await
17066            .expect("claim removal catalog.update succeeds");
17067        assert_eq!(
17068            forwarding.active_binding_count().unwrap(),
17069            1,
17070            "removing an attested target claim must leave the route census unchanged"
17071        );
17072        assert!(
17073            client_rx.try_recv().is_err(),
17074            "claim removal must not emit route.closed or a client route GOODBYE"
17075        );
17076        assert!(
17077            target_rx.try_recv().is_err(),
17078            "claim removal must not send the target a route GOODBYE"
17079        );
17080    }
17081
17082    /// A direct client may open a route to a denied capability provider; this
17083    /// policy applies only to attested supervised module origins, not to direct clients.
17084    #[tokio::test]
17085    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
17086        let registry = Arc::new(Registry::default());
17087        let forwarding = Arc::new(ForwardingTable::default());
17088        let supervisor = SupervisorHandle::new();
17089        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17090        let handler =
17091            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17092                .with_supervisor(supervisor);
17093        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
17094        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
17095        register_capability_manifest(
17096            &handler,
17097            &target_ctx,
17098            &mut target_rx,
17099            capability_manifest("target", &["credentials-provider/v1"], &[]),
17100            1,
17101        )
17102        .await;
17103        register_capability_manifest(
17104            &handler,
17105            &opener_ctx,
17106            &mut opener_rx,
17107            capability_manifest("opener", &[], &["credentials-provider/v1"]),
17108            2,
17109        )
17110        .await;
17111
17112        let (_client_rx, bind) = open_route_for_capability_test(
17113            &handler,
17114            &target_ctx,
17115            &mut target_rx,
17116            742,
17117            3,
17118            "target",
17119            None,
17120        )
17121        .await;
17122        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17123            panic!("direct scope-honesty route must bind");
17124        };
17125        assert_eq!(principal, Some(Principal::Direct));
17126        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17127    }
17128
17129    /// A module that denies a capability receives no self-route exemption when it
17130    /// also attestedly provides that capability.
17131    #[tokio::test]
17132    async fn must_never_reach_self_route_is_capability_forbidden() {
17133        let registry = Arc::new(Registry::default());
17134        let forwarding = Arc::new(ForwardingTable::default());
17135        let supervisor = SupervisorHandle::new();
17136        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17137        let handler =
17138            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17139                .with_supervisor(supervisor);
17140        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17141        register_capability_manifest(
17142            &handler,
17143            &self_ctx,
17144            &mut self_rx,
17145            capability_manifest(
17146                "self-provider",
17147                &["credentials-provider/v1"],
17148                &["credentials-provider/v1"],
17149            ),
17150            1,
17151        )
17152        .await;
17153
17154        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17155        let replies = handler
17156            .handle_control_frame(
17157                &client_ctx,
17158                route_open_frame_with_admission_facts(
17159                    2,
17160                    "self-provider",
17161                    unique_project_root("admission-facts"),
17162                    Some(ConsumerIdentity {
17163                        module_id: "self-provider".to_string(),
17164                        launch_nonce: "self-nonce".to_string(),
17165                    }),
17166                    None,
17167                ),
17168            )
17169            .await
17170            .expect("self-route refusal returns a typed frame");
17171        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17172        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17173        assert!(
17174            self_rx.try_recv().is_err(),
17175            "self denial must not relay route.bind"
17176        );
17177    }
17178
17179    #[test]
17180    fn unsupported_channel_zero_frame_returns_error() {
17181        let handler = ControlHandler::default();
17182        let request = Frame::build(
17183            FrameType::Request,
17184            control_flags(),
17185            0,
17186            0,
17187            21,
17188            b"opaque".to_vec(),
17189        )
17190        .unwrap();
17191
17192        let response = handler
17193            .handle_control(ConnectionId::new(1), request)
17194            .unwrap();
17195
17196        assert_eq!(response[0].header.ty, FrameType::Error);
17197        assert_eq!(
17198            parse_error(&response[0])["code"],
17199            "unsupported_control_frame"
17200        );
17201    }
17202
17203    /// Blue/green swap at the control-plane boundary. The supervisor that opens
17204    /// a swap is not wired yet, so the candidate is registered here directly
17205    /// into the registry and forwarding candidate slots, the way the swap's
17206    /// HELLO admission will.
17207    mod swap {
17208        use super::*;
17209
17210        const INCUMBENT: ConnectionId = ConnectionId::new(30);
17211        const CANDIDATE: ConnectionId = ConnectionId::new(40);
17212
17213        struct Swap {
17214            registry: Arc<Registry>,
17215            forwarding: Arc<ForwardingTable>,
17216            handler: ControlHandler,
17217            incumbent_ctx: RouteCtx,
17218            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17219            candidate_ctx: RouteCtx,
17220            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17221        }
17222
17223        async fn swap_with_incumbent() -> Swap {
17224            let registry = Arc::new(Registry::default());
17225            let forwarding = Arc::new(ForwardingTable::default());
17226            let handler =
17227                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17228            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17229            hello_via_sink(
17230                &handler,
17231                &incumbent_ctx,
17232                &mut incumbent_rx,
17233                hello_frame("aft", PROTOCOL_VERSION, 7),
17234            )
17235            .await;
17236            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17237            Swap {
17238                registry,
17239                forwarding,
17240                handler,
17241                incumbent_ctx,
17242                incumbent_rx,
17243                candidate_ctx,
17244                candidate_rx,
17245            }
17246        }
17247
17248        fn register_candidate(swap: &Swap, ready: Option<bool>) {
17249            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17250            candidate_manifest.ready = ready;
17251            let registration = swap
17252                .registry
17253                .register_candidate_with_control_ops(
17254                    candidate_manifest,
17255                    PROTOCOL_VERSION,
17256                    CANDIDATE,
17257                    module_baseline_control_ops(),
17258                )
17259                .unwrap();
17260            swap.forwarding
17261                .register_candidate_module_connection(
17262                    CANDIDATE,
17263                    "aft".to_string(),
17264                    PROTOCOL_VERSION,
17265                    manifest_concurrency(&registration.manifest),
17266                    swap.candidate_ctx.egress.clone(),
17267                )
17268                .unwrap();
17269        }
17270
17271        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17272            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17273            swap.registry.promote_candidate("aft").unwrap().unwrap();
17274            cutover.incumbent.unwrap()
17275        }
17276
17277        /// A swap candidate's HELLO reserves its candidate registry entry
17278        /// before installing its candidate forwarding endpoint. The swap
17279        /// supervisor promotes a ready candidate as soon as it reads it as
17280        /// registered, and promotion cuts forwarding over to the candidate
17281        /// endpoint, so the lookup the supervisor reads
17282        /// (`Registry::published_registration`) must not return the candidate
17283        /// until that endpoint exists. Otherwise the cutover finds no candidate
17284        /// endpoint and the swap fails as if the candidate had disconnected.
17285        #[tokio::test]
17286        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17287            let mut swap = swap_with_incumbent().await;
17288            let hold = Arc::new(std::sync::Barrier::new(2));
17289            let handler = swap
17290                .handler
17291                .clone()
17292                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17293            handler
17294                .supervisor
17295                .open_swap("aft", "candidate-nonce".to_string());
17296            let hello = {
17297                let handler = handler.clone();
17298                let candidate_ctx = swap.candidate_ctx.clone();
17299                std::thread::spawn(move || {
17300                    handler
17301                        .handle_hello(
17302                            candidate_ctx.connection_id,
17303                            Some(candidate_ctx.egress.clone()),
17304                            hello_frame_with_nonce(
17305                                "aft",
17306                                PROTOCOL_VERSION,
17307                                8,
17308                                Some("candidate-nonce"),
17309                            ),
17310                        )
17311                        .unwrap()
17312                })
17313            };
17314
17315            hold.wait();
17316            // Parked: the candidate slot is claimed but not published, and
17317            // forwarding has no candidate endpoint.
17318            assert!(swap
17319                .registry
17320                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17321                .unwrap()
17322                .is_some());
17323            assert!(swap
17324                .registry
17325                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17326                .unwrap()
17327                .is_none());
17328            assert!(swap
17329                .forwarding
17330                .module_endpoint_for_connection(CANDIDATE)
17331                .unwrap()
17332                .is_none());
17333
17334            hold.wait();
17335            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17336            assert_eq!(
17337                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17338                FrameType::HelloAck
17339            );
17340            let published = swap
17341                .registry
17342                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17343                .unwrap()
17344                .expect("candidate is published once its HELLO completes");
17345            assert_eq!(published.connection_id, CANDIDATE);
17346            assert!(swap
17347                .forwarding
17348                .module_endpoint_for_connection(CANDIDATE)
17349                .unwrap()
17350                .is_some());
17351        }
17352
17353        fn keyed_total(counters: &Value, key: &str) -> u64 {
17354            counters[key]
17355                .as_object()
17356                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17357                .unwrap_or(0)
17358        }
17359
17360        #[tokio::test]
17361        async fn replacement_logs_old_end_and_new_admission_once() {
17362            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17363            let mut swap = swap_with_incumbent().await;
17364            swap.handler
17365                .supervisor
17366                .open_swap("aft", "candidate-nonce".to_string());
17367            hello_via_sink(
17368                &swap.handler,
17369                &swap.candidate_ctx,
17370                &mut swap.candidate_rx,
17371                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17372            )
17373            .await;
17374            cutover(&swap);
17375            swap.handler.cleanup_connection(INCUMBENT).unwrap();
17376
17377            let captured = crate::router::test_log::captured_logs(&logs);
17378            println!("captured replacement registry logs:\n{captured}");
17379            let new_admissions: Vec<_> = captured
17380                .lines()
17381                .filter(|line| {
17382                    line.contains("connection_id=40")
17383                        && line.contains("swap candidate registered; not routable until cutover")
17384                })
17385                .collect();
17386            assert_eq!(
17387                new_admissions.len(),
17388                1,
17389                "unexpected admission logs: {captured}"
17390            );
17391            assert!(new_admissions[0].contains("module_id=aft"));
17392            assert!(new_admissions[0].contains("connection_id=40"));
17393            assert!(
17394                new_admissions[0].contains("swap candidate registered; not routable until cutover")
17395            );
17396            assert!(new_admissions[0].contains("ready=true"));
17397
17398            let old_admissions: Vec<_> = captured
17399                .lines()
17400                .filter(|line| {
17401                    line.contains("module registered module_id=aft ")
17402                        && line.contains("connection_id=30")
17403                })
17404                .collect();
17405            assert_eq!(
17406                old_admissions.len(),
17407                1,
17408                "unexpected incumbent admission: {captured}"
17409            );
17410
17411            let old_ends: Vec<_> = captured
17412                .lines()
17413                .filter(|line| {
17414                    line.contains("connection_id=30") && line.contains("module registration ended")
17415                })
17416                .collect();
17417            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17418            assert!(old_ends[0].contains("module_id=aft"));
17419            assert!(old_ends[0].contains("connection_id=30"));
17420            assert!(old_ends[0].contains("reason=replaced"));
17421            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17422            assert_eq!(
17423                captured
17424                    .lines()
17425                    .filter(|line| line.contains("module registration promoted"))
17426                    .count(),
17427                1,
17428                "unexpected promotion logs: {captured}"
17429            );
17430        }
17431
17432        /// An ack from the incumbent for a bind it was sent before cutover,
17433        /// arriving before the incumbent is drained. The incumbent is the live
17434        /// connection carrying every other client's routes, so the ack must
17435        /// not end it: the waiting client is told to retry, the reservation is
17436        /// given back, and the incumbent is told to drop just that binding.
17437        #[tokio::test]
17438        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17439            let mut swap = swap_with_incumbent().await;
17440            let handler = swap.handler.clone();
17441
17442            // A co-tenant route, bound on the incumbent before the swap.
17443            let cotenant = ConnectionId::new(31);
17444            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17445            let (cotenant_task, cotenant_bind) = relay_route_open(
17446                &handler,
17447                cotenant,
17448                &cotenant_ctx.egress,
17449                &mut swap.incumbent_rx,
17450                100,
17451                "aft",
17452                "swap-cotenant",
17453            )
17454            .await;
17455            handler
17456                .handle_control_frame(
17457                    &swap.incumbent_ctx,
17458                    route_bind_ack(cotenant_bind.header.corr),
17459                )
17460                .await
17461                .unwrap();
17462            assert!(cotenant_task.await.unwrap().is_empty());
17463            let (cotenant_channel, cotenant_epoch) =
17464                published_route(&cotenant_rx.recv().await.unwrap());
17465
17466            // A second route.open, relayed to the incumbent and not yet acked.
17467            let caller = ConnectionId::new(32);
17468            let (caller_ctx, mut caller_rx) = route_ctx(caller);
17469            let (caller_task, caller_bind) = relay_route_open(
17470                &handler,
17471                caller,
17472                &caller_ctx.egress,
17473                &mut swap.incumbent_rx,
17474                101,
17475                "aft",
17476                "swap-caller",
17477            )
17478            .await;
17479            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17480
17481            register_candidate(&swap, None);
17482            cutover(&swap);
17483
17484            // The incumbent acks after promotion and before any drain.
17485            let ack = handler
17486                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17487                .await;
17488            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17489            if module_loop_error.is_some() {
17490                // What the connection loop does with an untranslated router
17491                // error: end the connection, releasing every route on it.
17492                handler.cleanup_connection(INCUMBENT).unwrap();
17493            }
17494
17495            // 1. The incumbent's other routes survive.
17496            assert!(
17497                cotenant_rx.try_recv().is_err(),
17498                "the co-tenant route on the incumbent was torn down by one late ack: \
17499                 {module_loop_error:?}"
17500            );
17501            assert!(matches!(
17502                swap.forwarding
17503                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17504                    .unwrap(),
17505                DataRoute::Client(DataRouteState::Bound(_))
17506            ));
17507            assert_eq!(module_loop_error, None);
17508            assert!(swap
17509                .registry
17510                .get_module_by_connection(INCUMBENT)
17511                .unwrap()
17512                .is_some());
17513
17514            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
17515            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17516                .await
17517                .expect("the incumbent is told to drop the abandoned binding")
17518                .unwrap()
17519                .frame;
17520            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17521            assert_eq!(goodbye.header.channel, abandoned_channel);
17522            assert_eq!(goodbye.header.epoch, abandoned_epoch);
17523            assert!(swap.incumbent_rx.try_recv().is_err());
17524
17525            // 3. The waiting client gets a retryable refusal and no route.
17526            let response = caller_task.await.unwrap();
17527            assert_eq!(response.len(), 1);
17528            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17529            assert!(caller_rx.try_recv().is_err());
17530
17531            // 4. The reservation pair is given back, and the pending bind
17532            //    settled exactly once: one accepted open (the co-tenant) and one
17533            //    refused open (the caller), nothing counted twice.
17534            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17535            let counters = handler.counters().snapshot();
17536            assert_eq!(
17537                keyed_total(&counters, "route_open_accepted_by_principal"),
17538                1
17539            );
17540            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17541            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17542        }
17543
17544        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
17545        /// id would resolve to the promoted candidate and every new route.open
17546        /// would be refused as reloading, leaving neither process routable.
17547        #[tokio::test]
17548        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17549            let mut swap = swap_with_incumbent().await;
17550            register_candidate(&swap, None);
17551            let incumbent = cutover(&swap);
17552            swap.forwarding
17553                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17554                .unwrap()
17555                .expect("the incumbent is still registered");
17556
17557            let client = ConnectionId::new(33);
17558            let (client_ctx, mut client_rx) = route_ctx(client);
17559            let route_handler = swap.handler.clone();
17560            let open_ctx = RouteCtx {
17561                connection_id: client,
17562                egress: client_ctx.egress.clone(),
17563            };
17564            let mut route_task = tokio::spawn(async move {
17565                route_handler
17566                    .handle_control_frame(
17567                        &open_ctx,
17568                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17569                    )
17570                    .await
17571                    .unwrap()
17572            });
17573            let bind = tokio::select! {
17574                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17575                response = &mut route_task => {
17576                    let response = response.unwrap();
17577                    panic!(
17578                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
17579                        parse_error(&response[0])["code"]
17580                    );
17581                }
17582            };
17583            swap.handler
17584                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17585                .await
17586                .unwrap();
17587            assert!(route_task.await.unwrap().is_empty());
17588            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17589            match swap
17590                .forwarding
17591                .lookup_data_route(client, channel, epoch)
17592                .unwrap()
17593            {
17594                DataRoute::Client(DataRouteState::Bound(route)) => {
17595                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17596                }
17597                other => panic!("expected a bound route on the candidate, got {other:?}"),
17598            }
17599            assert!(swap.incumbent_rx.try_recv().is_err());
17600        }
17601
17602        /// A candidate declares itself ready with `catalog.update` on its own
17603        /// connection. If the connection-keyed registry lookups searched only the
17604        /// active slot, this would answer `not_registered` and the candidate
17605        /// would never become ready.
17606        #[tokio::test]
17607        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17608            let swap = swap_with_incumbent().await;
17609            register_candidate(&swap, Some(false));
17610            let update = Frame::build(
17611                FrameType::Request,
17612                control_flags(),
17613                0,
17614                0,
17615                55,
17616                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17617                    provides: manifest("aft", PROTOCOL_VERSION).provides,
17618                    capabilities: None,
17619                    ready: Some(true),
17620                })
17621                .unwrap(),
17622            )
17623            .unwrap();
17624
17625            let replies = swap
17626                .handler
17627                .handle_control_frame(&swap.candidate_ctx, update)
17628                .await
17629                .unwrap();
17630
17631            assert_eq!(replies.len(), 1);
17632            assert_eq!(
17633                replies[0].header.ty,
17634                FrameType::Response,
17635                "candidate catalog.update was refused: {:?}",
17636                serde_json::from_slice::<Value>(&replies[0].body).ok()
17637            );
17638            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17639            assert_eq!(
17640                swap.registry
17641                    .get_module("aft")
17642                    .unwrap()
17643                    .unwrap()
17644                    .connection_id,
17645                INCUMBENT
17646            );
17647        }
17648    }
17649
17650    /// The HELLO gate while the supervisor has a swap open: only the nonce it
17651    /// minted for the candidate admits a second process, into the candidate
17652    /// slot, and that check runs ahead of the reserved-module gate.
17653    mod swap_admission {
17654        use super::*;
17655
17656        const INCUMBENT_NONCE: &str = "incumbent-nonce";
17657        const CANDIDATE_NONCE: &str = "candidate-nonce";
17658
17659        fn handler_with_incumbent(
17660            module_id: &str,
17661            reserved: bool,
17662        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17663            let registry = Arc::new(Registry::default());
17664            let supervisor = SupervisorHandle::new();
17665            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17666            if reserved {
17667                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17668            }
17669            let handler =
17670                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
17671            let incumbent = handler
17672                .handle_control(
17673                    ConnectionId::new(1),
17674                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17675                )
17676                .unwrap();
17677            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17678            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17679            (registry, supervisor, handler)
17680        }
17681
17682        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
17683        /// admits every nonce, so while a swap is open the swap gate is the only
17684        /// thing between a key-holder and the candidate slot. A nonce the
17685        /// supervisor did not mint, or none at all, is refused, and neither the
17686        /// incumbent's registration nor the candidate slot moves.
17687        #[test]
17688        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17689            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17690
17691            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17692                let replies = handler
17693                    .handle_control(
17694                        ConnectionId::new(connection),
17695                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17696                    )
17697                    .unwrap();
17698                assert_eq!(replies[0].header.ty, FrameType::Error);
17699                assert_eq!(
17700                    parse_error(&replies[0])["code"],
17701                    "swap_token_invalid",
17702                    "nonce {nonce:?}"
17703                );
17704            }
17705            assert!(registry.get_candidate("aft").unwrap().is_none());
17706            assert_eq!(
17707                registry.get_module("aft").unwrap().unwrap().connection_id,
17708                ConnectionId::new(1)
17709            );
17710
17711            // Control: the minted token is admitted, into the candidate slot,
17712            // and only once.
17713            let admitted = handler
17714                .handle_control(
17715                    ConnectionId::new(4),
17716                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17717                )
17718                .unwrap();
17719            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17720            assert_eq!(
17721                registry
17722                    .get_candidate("aft")
17723                    .unwrap()
17724                    .unwrap()
17725                    .connection_id,
17726                ConnectionId::new(4)
17727            );
17728            assert_eq!(
17729                registry.get_module("aft").unwrap().unwrap().connection_id,
17730                ConnectionId::new(1),
17731                "the candidate must not take the active slot"
17732            );
17733            let replayed = handler
17734                .handle_control(
17735                    ConnectionId::new(5),
17736                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17737                )
17738                .unwrap();
17739            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17740
17741            // The case only this gate covers: the incumbent has died mid-swap,
17742            // so its duplicate refusal is gone too, and without the gate a
17743            // key-holder would take the id's ACTIVE slot.
17744            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17745            let squatter = handler
17746                .handle_control(
17747                    ConnectionId::new(6),
17748                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17749                )
17750                .unwrap();
17751            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17752            assert!(
17753                registry.get_module("aft").unwrap().is_none(),
17754                "a squatter took the active slot of an id being swapped"
17755            );
17756        }
17757
17758        /// Design mutation arm (iii). A reserved module's candidate presents a
17759        /// nonce the reserved gate has never seen (that gate holds the
17760        /// incumbent's), so the swap gate must run first or the candidate is
17761        /// refused `reserved_module` and a reserved module can never be swapped.
17762        #[test]
17763        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17764            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17765
17766            let replies = handler
17767                .handle_control(
17768                    ConnectionId::new(2),
17769                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17770                )
17771                .unwrap();
17772
17773            assert_eq!(
17774                replies[0].header.ty,
17775                FrameType::HelloAck,
17776                "reserved candidate refused: {:?}",
17777                serde_json::from_slice::<Value>(&replies[0].body).ok()
17778            );
17779            assert_eq!(
17780                registry
17781                    .get_candidate("vault")
17782                    .unwrap()
17783                    .unwrap()
17784                    .connection_id,
17785                ConnectionId::new(2)
17786            );
17787        }
17788
17789        /// With no swap open the gate is inert: the incumbent's reserved gate
17790        /// and duplicate refusal behave exactly as before.
17791        #[test]
17792        fn without_an_open_swap_the_ordinary_gates_decide() {
17793            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17794            supervisor.close_swap("vault");
17795
17796            let candidate = handler
17797                .handle_control(
17798                    ConnectionId::new(2),
17799                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17800                )
17801                .unwrap();
17802            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17803            let duplicate = handler
17804                .handle_control(
17805                    ConnectionId::new(3),
17806                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17807                )
17808                .unwrap();
17809            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17810            assert!(registry.get_candidate("vault").unwrap().is_none());
17811        }
17812    }
17813
17814    /// `scope.sync` and `scope.describe` through the real control handler: who
17815    /// may sync is decided by the registration and launch nonce of the module
17816    /// connection, never by the request body.
17817    mod scopes {
17818        mod idle_tests {
17819            use super::*;
17820            include!("scope_idle_tests.rs");
17821        }
17822        mod route_open_timings {
17823            use super::*;
17824            include!("route_open_timing_tests.rs");
17825        }
17826        mod route_open_refusal_log {
17827            use super::*;
17828            include!("route_open_refusal_log_tests.rs");
17829        }
17830        mod activity {
17831            include!("activity_scope_tests.rs");
17832        }
17833        mod delta {
17834            include!("scope_handler_delta_tests.rs");
17835        }
17836        mod call_modes {
17837            include!("scope_call_mode_tests.rs");
17838        }
17839        mod federated {
17840            include!("scope_federated_tests.rs");
17841        }
17842        #[tokio::test]
17843        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17844            let rig = rig().await;
17845            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17846                "s".into(),
17847                1,
17848                "tool".into(),
17849            ))
17850            .unwrap();
17851            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17852            let (direct, _rx) = wide_ctx(999);
17853            for (ctx, expected) in [
17854                (&rig.owner, FrameType::Response),
17855                (&direct, FrameType::Error),
17856            ] {
17857                let frame =
17858                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17859                        .unwrap();
17860                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17861                assert_eq!(replies[0].header.ty, expected);
17862                if expected == FrameType::Response {
17863                    let reply: subc_control::ScopeServedByReply =
17864                        serde_json::from_slice(&replies[0].body).unwrap();
17865                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17866                } else {
17867                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17868                }
17869            }
17870            let reply = call(
17871                &rig.handler,
17872                &rig.owner,
17873                &ModuleControlRequestFromModule::ScopeDescribe {
17874                    owner: Principal::Reserved {
17875                        module_id: OWNER.into(),
17876                    },
17877                    scope_ref: "s".into(),
17878                },
17879            )
17880            .await;
17881            assert!(matches!(
17882                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17883                ModuleControlResponseToModule::ScopeDescribe { .. }
17884            ));
17885        }
17886
17887        use subc_protocol::scope::{
17888            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17889            ScopeStatus,
17890        };
17891
17892        use super::*;
17893
17894        const OWNER: &str = "prefrontal-core";
17895
17896        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17897            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17898        }
17899
17900        async fn call(
17901            handler: &ControlHandler,
17902            ctx: &RouteCtx,
17903            request: &ModuleControlRequestFromModule,
17904        ) -> Frame {
17905            let body = serde_json::to_vec(request).unwrap();
17906            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17907            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17908            assert_eq!(replies.len(), 1, "{replies:?}");
17909            replies.pop().unwrap()
17910        }
17911
17912        async fn sync(
17913            handler: &ControlHandler,
17914            ctx: &RouteCtx,
17915            generation: u64,
17916            scopes: Vec<ScopeRecord>,
17917        ) -> Result<ModuleControlResponseToModule, String> {
17918            let reply = call(
17919                handler,
17920                ctx,
17921                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17922            )
17923            .await;
17924            match reply.header.ty {
17925                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17926                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17927            }
17928        }
17929
17930        async fn describe(
17931            handler: &ControlHandler,
17932            ctx: &RouteCtx,
17933            owner: &str,
17934            scope_ref: &str,
17935        ) -> ModuleControlResponseToModule {
17936            let reply = call(
17937                handler,
17938                ctx,
17939                &ModuleControlRequestFromModule::ScopeDescribe {
17940                    owner: Principal::Reserved {
17941                        module_id: owner.to_string(),
17942                    },
17943                    scope_ref: scope_ref.to_string(),
17944                },
17945            )
17946            .await;
17947            assert_eq!(
17948                reply.header.ty,
17949                FrameType::Response,
17950                "{:?}",
17951                parse_error(&reply)
17952            );
17953            serde_json::from_slice(&reply.body).unwrap()
17954        }
17955
17956        #[tokio::test]
17957        async fn scope_describe_never_echoes_bind_proof_or_hash() {
17958            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17959            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17960            let mut rig = rig().await;
17961            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17962                .await;
17963            let _bound = rig
17964                .bound(
17965                    Some(AFT),
17966                    PLEXUS,
17967                    Some(
17968                        rig.selector("s", Some(1))
17969                            .with_bind_proof(Some(PROOF.into())),
17970                    ),
17971                )
17972                .await;
17973            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17974            let output = serde_json::to_string(&reply).unwrap();
17975            assert!(output.contains("\"status\":\"live\""));
17976            assert!(!output.contains(PROOF));
17977            assert!(!output.contains(HASH));
17978        }
17979
17980        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
17981        async fn module(
17982            handler: &ControlHandler,
17983            connection: u64,
17984            module_id: &str,
17985            nonce: Option<&str>,
17986        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17987            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17988            hello_via_sink(
17989                handler,
17990                &ctx,
17991                &mut rx,
17992                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17993            )
17994            .await;
17995            (ctx, rx)
17996        }
17997
17998        /// `direct` and every other client connection has no registration, so
17999        /// it can neither sync nor own a scope.
18000        #[tokio::test]
18001        async fn a_client_connection_cannot_sync_or_describe() {
18002            let handler = ControlHandler::new(Arc::new(Registry::default()));
18003            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
18004            for request in [
18005                ModuleControlRequestFromModule::ScopeSync {
18006                    generation: 1,
18007                    scopes: vec![head("s", 1)],
18008                },
18009                ModuleControlRequestFromModule::ScopeDescribe {
18010                    owner: Principal::Direct,
18011                    scope_ref: "s".to_string(),
18012                },
18013            ] {
18014                let reply = call(&handler, &ctx, &request).await;
18015                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
18016            }
18017            assert!(
18018                !handler
18019                    .scopes
18020                    .read()
18021                    .unwrap()
18022                    .describe(
18023                        &Principal::Reserved {
18024                            module_id: OWNER.to_string()
18025                        },
18026                        "s"
18027                    )
18028                    .owner_synced
18029            );
18030        }
18031
18032        /// A module the supervisor did not spawn registers without a launch
18033        /// nonce, so it is never an owner's current launch.
18034        #[tokio::test]
18035        async fn a_module_without_a_supervised_launch_cannot_sync() {
18036            let handler = ControlHandler::new(Arc::new(Registry::default()));
18037            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
18038            assert_eq!(
18039                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
18040                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18041            );
18042        }
18043
18044        #[tokio::test]
18045        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
18046            let supervisor = SupervisorHandle::new();
18047            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18048            let handler = ControlHandler::new(Arc::new(Registry::default()))
18049                .with_supervisor(supervisor.clone());
18050            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18051            sync(&handler, &incumbent, 1, vec![head("s", 1)])
18052                .await
18053                .expect("the current launch syncs");
18054
18055            // A swap candidate registers with the swap token and is refused
18056            // while the incumbent keeps syncing.
18057            supervisor.open_swap(OWNER, "n2".to_string());
18058            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
18059            assert_eq!(
18060                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18061                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18062            );
18063            sync(&handler, &incumbent, 2, vec![head("s", 1)])
18064                .await
18065                .expect("the serving owner syncs during the swap");
18066
18067            // The swap fails and is rolled back. The candidate never held sync
18068            // authority, and still cannot sync.
18069            supervisor.close_swap(OWNER);
18070            assert_eq!(
18071                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18072                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18073            );
18074            sync(&handler, &incumbent, 3, vec![head("s", 1)])
18075                .await
18076                .expect("the serving owner syncs after the rollback");
18077            handler.cleanup_connection(candidate.connection_id).unwrap();
18078
18079            // A swap that cuts over. Promotion records the candidate's nonce as
18080            // the module's spawn nonce, which is what `set_spawn_nonce` does
18081            // here; the promoted connection then takes authority at any
18082            // generation and the superseded incumbent is refused.
18083            supervisor.open_swap(OWNER, "n3".to_string());
18084            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
18085            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
18086            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
18087                .await
18088                .expect("the promoted launch takes authority");
18089            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18090                panic!("unexpected reply {reply:?}");
18091            };
18092            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
18093            assert_eq!(
18094                sync(&handler, &incumbent, 4, Vec::new()).await,
18095                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18096            );
18097        }
18098
18099        /// Authority dies with its connection: the cleanup path releases it,
18100        /// so the owner's next connection takes it at any generation.
18101        #[tokio::test]
18102        async fn closing_the_authority_connection_frees_sync_authority() {
18103            let supervisor = SupervisorHandle::new();
18104            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18105            let handler = ControlHandler::new(Arc::new(Registry::default()))
18106                .with_supervisor(supervisor.clone());
18107            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18108            sync(&handler, &first, 10, vec![head("s", 1)])
18109                .await
18110                .unwrap();
18111            handler.cleanup_connection(first.connection_id).unwrap();
18112
18113            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
18114            sync(&handler, &second, 1, vec![head("s", 1)])
18115                .await
18116                .expect("the next connection takes the released authority");
18117        }
18118
18119        #[tokio::test]
18120        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18121            let supervisor = SupervisorHandle::new();
18122            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18123            let handler =
18124                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18125            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18126            sync(&handler, &first, 10, vec![head("s", 1)])
18127                .await
18128                .unwrap();
18129            handler
18130                .handle_control_frame(
18131                    &first,
18132                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18133                )
18134                .await
18135                .unwrap();
18136            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18137            sync(&handler, &second, 1, vec![head("s", 1)])
18138                .await
18139                .expect("GOODBYE releases authority even if the old socket remains open");
18140        }
18141
18142        #[tokio::test]
18143        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18144            let registry = Arc::new(Registry::default());
18145            let supervisor_handle = SupervisorHandle::new();
18146            let supervisor =
18147                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
18148                    .with_handle(supervisor_handle.clone())
18149                    .with_daemon_incarnation("incarnation-7".to_string());
18150            // Configured with enabled: false, so the supervisor lists the
18151            // module without spawning a process for it.
18152            supervisor
18153                .supervise_configured(
18154                    ModuleSpec {
18155                        carry_containment: false,
18156                        module_id: OWNER.to_string(),
18157                        program: PathBuf::from("/nonexistent/prefrontal-core"),
18158                        args: Vec::new(),
18159                        env: Vec::new(),
18160                        reserved: false,
18161                        reserved_prefixes: Vec::new(),
18162                        protocol: ModuleProtocol::Subc,
18163                        overlap: Default::default(),
18164                    },
18165                    false,
18166                )
18167                .unwrap();
18168            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18169            let handler =
18170                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
18171            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18172
18173            // Configured but not yet synced: a reader waits for the owner.
18174            let ModuleControlResponseToModule::ScopeDescribe {
18175                status,
18176                daemon_incarnation,
18177                owner_synced,
18178                owner_configured,
18179                scope,
18180                ..
18181            } = describe(&handler, &reader, OWNER, "s").await
18182            else {
18183                panic!("not a describe reply");
18184            };
18185            assert_eq!(status, ScopeStatus::NotLive);
18186            assert_eq!(daemon_incarnation, "incarnation-7");
18187            assert!(!owner_synced);
18188            assert!(owner_configured);
18189            assert!(scope.is_none());
18190
18191            // Not a supervised module: the owner will never sync, and a reader
18192            // refuses rather than waits.
18193            let ModuleControlResponseToModule::ScopeDescribe {
18194                status,
18195                owner_configured,
18196                ..
18197            } = describe(&handler, &reader, "ghost", "s").await
18198            else {
18199                panic!("not a describe reply");
18200            };
18201            assert_eq!(status, ScopeStatus::NotLive);
18202            assert!(!owner_configured);
18203
18204            // Live, with the stamp fields and the computed owner_authorized.
18205            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18206            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18207            let ModuleControlResponseToModule::ScopeDescribe {
18208                status,
18209                scope_epoch,
18210                owner_synced,
18211                scope,
18212                ..
18213            } = describe(&handler, &reader, OWNER, "s").await
18214            else {
18215                panic!("not a describe reply");
18216            };
18217            assert_eq!(status, ScopeStatus::Live);
18218            assert_eq!(scope_epoch, Some(4));
18219            assert!(owner_synced);
18220            let stamp = scope.expect("a live scope carries its stamp");
18221            assert!(
18222                stamp.owner_authorized,
18223                "prefrontal-core is the default authority"
18224            );
18225            assert_eq!(stamp.kind, ScopeKind::Head);
18226        }
18227
18228        #[tokio::test]
18229        async fn scope_authority_owners_decides_owner_authorized() {
18230            let supervisor = SupervisorHandle::new();
18231            supervisor.set_spawn_nonce("broca", "b1".to_string());
18232            let handler = ControlHandler::new(Arc::new(Registry::default()))
18233                .with_supervisor(supervisor)
18234                .with_scope_authority_owners(vec!["broca".to_string()]);
18235            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18236            let mut gated = head("s", 1);
18237            gated.attributes.agent_id = Some("agent".to_string());
18238            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18239            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18240                describe(&handler, &broca, "broca", "s").await
18241            else {
18242                panic!("not a describe reply");
18243            };
18244            assert!(scope.unwrap().owner_authorized);
18245        }
18246
18247        /// With route admission, the stamp, the commit re-check and drains in
18248        /// place, the feature is advertised: the module ops in HELLO_ACK, and
18249        /// `scopes/v1` in HELLO_ACK and `server.describe`.
18250        #[tokio::test]
18251        async fn scope_ops_and_the_scopes_capability_are_advertised() {
18252            let handler = ControlHandler::new(Arc::new(Registry::default()));
18253            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18254            let ack = hello_via_sink(
18255                &handler,
18256                &ctx,
18257                &mut rx,
18258                hello_frame("m", PROTOCOL_VERSION, 1),
18259            )
18260            .await;
18261            let ack = parse_ack(&ack);
18262            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18263                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18264            }
18265            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18266
18267            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18268            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18269            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18270            let reply = handler
18271                .handle_control_frame(&client, frame)
18272                .await
18273                .unwrap()
18274                .pop()
18275                .unwrap();
18276            let ClientControlResponse::ServerDescribe { capabilities, .. } =
18277                serde_json::from_slice(&reply.body).unwrap()
18278            else {
18279                panic!("not a server.describe reply");
18280            };
18281            assert!(
18282                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18283                "{capabilities:?}"
18284            );
18285        }
18286
18287        // ---- route admission, stamps, commit re-check and drains ----------
18288
18289        const PLEXUS: &str = "plexus";
18290        const OTHER: &str = "other";
18291        const AFT: &str = "aft";
18292        const BROCA: &str = "broca";
18293        const MAGIC: &str = "magic-context";
18294
18295        fn nonce(module_id: &str) -> String {
18296            format!("nonce-{module_id}")
18297        }
18298
18299        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18300            let (tx, rx) = mpsc::channel(64);
18301            (
18302                RouteCtx {
18303                    connection_id: ConnectionId::new(connection),
18304                    egress: FrameSink::new(tx),
18305                },
18306                rx,
18307            )
18308        }
18309
18310        /// A daemon with a configured owner (prefrontal-core) registered on its
18311        /// own module connection, two routable targets (plexus, other), and
18312        /// launch nonces minted for the modules that open routes as carriers.
18313        struct Rig {
18314            handler: ControlHandler,
18315            forwarding: Arc<ForwardingTable>,
18316            owner: RouteCtx,
18317            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18318            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18319            generation: u64,
18320            next_connection: u64,
18321            _supervisor: Supervisor,
18322        }
18323
18324        async fn rig() -> Rig {
18325            rig_with_flow_support(true).await
18326        }
18327
18328        async fn rig_with_flow_support(flow_support: bool) -> Rig {
18329            rig_with_scope_capabilities(flow_support, false).await
18330        }
18331
18332        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18333            let mut provides = Vec::new();
18334            if flow_support {
18335                provides.push("flow-scopes/v1");
18336            }
18337            if run_support {
18338                provides.push("agent-run-scopes/v1");
18339            }
18340            rig_with_target_capabilities(&provides).await
18341        }
18342
18343        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18344            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18345            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18346            // Decoder versions alone cannot admit scoped calls: only a capability
18347            // declaration promises the required run or flow behaviour.
18348            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18349            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18350            Frame::build(
18351                FrameType::Hello,
18352                control_flags(),
18353                0,
18354                0,
18355                connection,
18356                serde_json::to_vec(&body).unwrap(),
18357            )
18358            .unwrap()
18359        }
18360
18361        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18362            let registry = Arc::new(Registry::default());
18363            let forwarding = Arc::new(ForwardingTable::default());
18364            let supervisor_handle = SupervisorHandle::new();
18365            let supervisor =
18366                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
18367                    .with_handle(supervisor_handle.clone());
18368            supervisor
18369                .supervise_configured(
18370                    ModuleSpec {
18371                        carry_containment: false,
18372                        module_id: OWNER.to_string(),
18373                        program: PathBuf::from("/nonexistent/prefrontal-core"),
18374                        args: Vec::new(),
18375                        env: Vec::new(),
18376                        reserved: false,
18377                        reserved_prefixes: Vec::new(),
18378                        protocol: ModuleProtocol::Subc,
18379                        overlap: Default::default(),
18380                    },
18381                    false,
18382                )
18383                .unwrap();
18384            for module_id in [OWNER, AFT, BROCA, MAGIC] {
18385                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18386            }
18387            let handler =
18388                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18389                    .with_supervisor(supervisor_handle);
18390            let (owner, mut owner_rx) = wide_ctx(1);
18391            hello_via_sink(
18392                &handler,
18393                &owner,
18394                &mut owner_rx,
18395                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18396            )
18397            .await;
18398            let mut modules = BTreeMap::new();
18399            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18400                let (ctx, mut rx) = wide_ctx(connection);
18401                let hello = hello_with_capabilities(module_id, connection, provides);
18402                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18403                modules.insert(module_id.to_string(), (ctx, rx));
18404            }
18405            Rig {
18406                handler,
18407                forwarding,
18408                owner,
18409                _owner_rx: owner_rx,
18410                modules,
18411                generation: 0,
18412                next_connection: 100,
18413                _supervisor: supervisor,
18414            }
18415        }
18416
18417        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18418            ScopeCarrier::new(Principal::Reserved {
18419                module_id: module_id.to_string(),
18420            })
18421            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18422        }
18423
18424        /// The scope most tests open under: aft carries to any module, broca
18425        /// only to plexus and other, and the owner delegates as agent-1.
18426        fn session(scope_epoch: u64) -> ScopeRecord {
18427            let mut record = head("s", scope_epoch);
18428            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18429            record.attributes.agent_id = Some("agent-1".to_string());
18430            record.attributes.delegates = true;
18431            record
18432        }
18433
18434        impl Rig {
18435            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18436                self.generation += 1;
18437                sync(&self.handler, &self.owner, self.generation, scopes)
18438                    .await
18439                    .expect("the owner's sync is accepted");
18440            }
18441
18442            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18443                ScopeSelector::new(
18444                    Principal::Reserved {
18445                        module_id: OWNER.to_string(),
18446                    },
18447                    scope_ref.to_string(),
18448                    scope_epoch,
18449                )
18450            }
18451
18452            fn open_frame(
18453                &mut self,
18454                opener: Option<&str>,
18455                target: &str,
18456                scope: Option<ScopeSelector>,
18457            ) -> (
18458                RouteCtx,
18459                mpsc::Receiver<crate::router::OutboundFrame>,
18460                Frame,
18461            ) {
18462                self.next_connection += 1;
18463                let (ctx, rx) = wide_ctx(self.next_connection);
18464                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18465                    busy_refusals: None,
18466                    target: RouteTarget::ToolProvider {
18467                        module_id: target.to_string(),
18468                    },
18469                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18470                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
18471                        module_id: module_id.to_string(),
18472                        launch_nonce: nonce(module_id),
18473                    }),
18474                    consumer_capabilities: None,
18475                    role_versions: None,
18476                    admission_facts: None,
18477                    scope,
18478                })
18479                .unwrap();
18480                let frame = Frame::build(
18481                    FrameType::Request,
18482                    control_flags(),
18483                    0,
18484                    0,
18485                    self.next_connection,
18486                    body,
18487                )
18488                .unwrap();
18489                (ctx, rx, frame)
18490            }
18491
18492            /// Open and expect a refusal before anything is relayed.
18493            async fn refused(
18494                &mut self,
18495                opener: Option<&str>,
18496                target: &str,
18497                scope: Option<ScopeSelector>,
18498            ) -> String {
18499                self.refusal_body(opener, target, scope).await["code"]
18500                    .as_str()
18501                    .unwrap()
18502                    .to_string()
18503            }
18504
18505            async fn refusal_body(
18506                &mut self,
18507                opener: Option<&str>,
18508                target: &str,
18509                scope: Option<ScopeSelector>,
18510            ) -> Value {
18511                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18512                let replies = tokio::time::timeout(
18513                    Duration::from_secs(2),
18514                    self.handler.handle_control_frame(&ctx, frame),
18515                )
18516                .await
18517                .expect("the open must be refused before waiting for a bind ack")
18518                .unwrap();
18519                assert_eq!(replies.len(), 1, "{replies:?}");
18520                assert_eq!(replies[0].header.ty, FrameType::Error);
18521                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18522                assert!(
18523                    module_rx.try_recv().is_err(),
18524                    "a refused open relays nothing"
18525                );
18526                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18527                parse_error(&replies[0])
18528            }
18529
18530            /// Start an open and return its task and the bind the target got.
18531            async fn relayed(
18532                &mut self,
18533                opener: Option<&str>,
18534                target: &str,
18535                scope: Option<ScopeSelector>,
18536            ) -> Relayed {
18537                self.relayed_with_legacy_root(opener, target, scope, None)
18538                    .await
18539            }
18540
18541            async fn relayed_with_legacy_root(
18542                &mut self,
18543                opener: Option<&str>,
18544                target: &str,
18545                scope: Option<ScopeSelector>,
18546                legacy_root: Option<&std::path::Path>,
18547            ) -> Relayed {
18548                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18549                if let Some(root) = legacy_root {
18550                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18551                    body["identity"]["project_root"] = serde_json::json!(root);
18552                    frame.body = serde_json::to_vec(&body).unwrap();
18553                }
18554                let handler = self.handler.clone();
18555                let task_ctx = ctx.clone();
18556                let task = tokio::spawn(async move {
18557                    handler
18558                        .handle_control_frame(&task_ctx, frame)
18559                        .await
18560                        .unwrap()
18561                });
18562                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18563                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18564                    .await
18565                    .expect("the target receives the relayed route.bind")
18566                    .unwrap()
18567                    .frame;
18568                Relayed {
18569                    target: target.to_string(),
18570                    client: ctx,
18571                    client_rx: rx,
18572                    task,
18573                    bind,
18574                }
18575            }
18576
18577            async fn ack(&self, relayed: &Relayed) {
18578                let (module, _) = &self.modules[&relayed.target];
18579                self.handler
18580                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18581                    .await
18582                    .unwrap();
18583            }
18584
18585            /// Open, ack and return the bound route.
18586            async fn bound(
18587                &mut self,
18588                opener: Option<&str>,
18589                target: &str,
18590                scope: Option<ScopeSelector>,
18591            ) -> Bound {
18592                let relayed = self.relayed(opener, target, scope).await;
18593                self.commit_relayed(relayed).await
18594            }
18595
18596            async fn bound_with_legacy_root(
18597                &mut self,
18598                opener: Option<&str>,
18599                target: &str,
18600                scope: Option<ScopeSelector>,
18601                root: &std::path::Path,
18602            ) -> Bound {
18603                let relayed = self
18604                    .relayed_with_legacy_root(opener, target, scope, Some(root))
18605                    .await;
18606                self.commit_relayed(relayed).await
18607            }
18608
18609            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18610                self.ack(&relayed).await;
18611                let Relayed {
18612                    target,
18613                    client,
18614                    mut client_rx,
18615                    task,
18616                    bind,
18617                } = relayed;
18618                assert!(
18619                    task.await.unwrap().is_empty(),
18620                    "the open is answered by commit"
18621                );
18622                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18623                Bound {
18624                    target,
18625                    client,
18626                    client_rx,
18627                    channel,
18628                    epoch,
18629                    bind,
18630                }
18631            }
18632
18633            fn live(&self, route: &Bound) -> bool {
18634                matches!(
18635                    self.forwarding
18636                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18637                        .unwrap(),
18638                    DataRoute::Client(DataRouteState::Bound(_))
18639                )
18640            }
18641        }
18642
18643        struct Relayed {
18644            target: String,
18645            client: RouteCtx,
18646            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18647            task: tokio::task::JoinHandle<Vec<Frame>>,
18648            bind: Frame,
18649        }
18650
18651        struct Bound {
18652            target: String,
18653            client: RouteCtx,
18654            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18655            channel: u16,
18656            epoch: u32,
18657            bind: Frame,
18658        }
18659
18660        impl Bound {
18661            /// The reason of the `route.closed` this client was sent, after
18662            /// checking it also got a GOODBYE on exactly this route.
18663            fn closed_reason(&mut self) -> RouteCloseReason {
18664                let mut reason = None;
18665                let mut goodbye = false;
18666                while let Ok(outbound) = self.client_rx.try_recv() {
18667                    let frame = outbound.frame;
18668                    match frame.header.ty {
18669                        FrameType::Goodbye => {
18670                            assert_eq!(
18671                                (frame.header.channel, frame.header.epoch),
18672                                (self.channel, self.epoch)
18673                            );
18674                            goodbye = true;
18675                        }
18676                        FrameType::Push => {
18677                            let ClientControlPush::RouteClosed {
18678                                reason: r,
18679                                module_id,
18680                                ..
18681                            } = serde_json::from_slice(&frame.body).unwrap()
18682                            else {
18683                                panic!("unexpected push");
18684                            };
18685                            assert_eq!(module_id, self.target);
18686                            reason = Some(r);
18687                        }
18688                        other => panic!("unexpected frame {other:?}"),
18689                    }
18690                }
18691                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18692                reason.expect("the client is told why the route closed")
18693            }
18694
18695            fn untouched(&mut self) -> bool {
18696                self.client_rx.try_recv().is_err()
18697            }
18698
18699            fn stamp(&self) -> Option<ScopeStamp> {
18700                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18701                    ModuleControlRequest::RouteBind { scope, .. } => scope,
18702                    other => panic!("expected a route.bind, got {other:?}"),
18703                }
18704            }
18705        }
18706
18707        #[tokio::test]
18708        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18709        ) {
18710            let mut rig = rig().await;
18711            let mut record = session(1);
18712            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18713            record.child_owners = vec![Principal::Reserved {
18714                module_id: MAGIC.to_string(),
18715            }];
18716            rig.sync(vec![record]).await;
18717            let scope = || Some(rig_selector("s", Some(1)));
18718
18719            // Admitted: the owner, a bare carrier to any module, a targeted
18720            // carrier to its listed module.
18721            rig.bound(Some(OWNER), PLEXUS, scope()).await;
18722            rig.bound(Some(AFT), OTHER, scope()).await;
18723            rig.bound(Some(BROCA), PLEXUS, scope()).await;
18724
18725            // Refused scope_not_carrier: a targeted carrier to an unlisted
18726            // module, a module that is not listed at all (a child owner is not
18727            // a carrier), and a direct key-holder.
18728            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18729                assert_eq!(
18730                    rig.refused(opener, target, scope()).await,
18731                    error_codes::SCOPE_NOT_CARRIER,
18732                    "{opener:?} -> {target}"
18733                );
18734            }
18735        }
18736
18737        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18738            ScopeSelector::new(
18739                Principal::Reserved {
18740                    module_id: OWNER.to_string(),
18741                },
18742                scope_ref.to_string(),
18743                scope_epoch,
18744            )
18745        }
18746
18747        #[tokio::test]
18748        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18749            let mut rig = rig().await;
18750            rig.sync(vec![session(1)]).await;
18751            for opener in [OWNER, AFT] {
18752                assert_eq!(
18753                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18754                        .await,
18755                    error_codes::SCOPE_EPOCH_REQUIRED,
18756                    "{opener}"
18757                );
18758            }
18759        }
18760
18761        #[tokio::test]
18762        async fn admission_separates_not_synced_not_live_and_ended() {
18763            let mut rig = rig().await;
18764            // Before the configured owner's first sync: retryable.
18765            let code = rig
18766                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18767                .await;
18768            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18769            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18770
18771            // An owner that is not configured will never sync: terminal.
18772            let ghost = ScopeSelector::new(
18773                Principal::Reserved {
18774                    module_id: "ghost".to_string(),
18775                },
18776                "s".to_string(),
18777                Some(1),
18778            );
18779            assert_eq!(
18780                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18781                error_codes::SCOPE_NOT_LIVE
18782            );
18783
18784            rig.sync(vec![session(2)]).await;
18785            assert_eq!(
18786                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18787                    .await,
18788                error_codes::SCOPE_NOT_LIVE
18789            );
18790            for epoch in [1, 3] {
18791                assert_eq!(
18792                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18793                        .await,
18794                    error_codes::SCOPE_ENDED,
18795                    "epoch {epoch}"
18796                );
18797            }
18798            // Control: the live epoch is admitted.
18799            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18800                .await;
18801        }
18802
18803        #[tokio::test]
18804        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18805            let mut rig = rig().await;
18806            rig.sync(vec![session(1)]).await;
18807            let route = rig
18808                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18809                .await;
18810            let stamp = route.stamp().expect("a scoped bind carries the stamp");
18811            assert_eq!(stamp.scope_ref, "s");
18812            assert_eq!(stamp.scope_epoch, 1);
18813            assert_eq!(stamp.kind, ScopeKind::Head);
18814            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18815            assert!(stamp.attributes.delegates);
18816            assert!(stamp.owner_authorized);
18817            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18818            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18819
18820            // broca owns a scope of its own on its own module connection; it is
18821            // not in scope_authority_owners, so its stamp is not authorized.
18822            let (broca, mut broca_rx) = wide_ctx(50);
18823            hello_via_sink(
18824                &rig.handler,
18825                &broca,
18826                &mut broca_rx,
18827                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18828            )
18829            .await;
18830            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18831                .await
18832                .unwrap();
18833            let own = ScopeSelector::new(
18834                Principal::Reserved {
18835                    module_id: BROCA.to_string(),
18836                },
18837                "b".to_string(),
18838                Some(1),
18839            );
18840            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18841            assert!(!route.stamp().unwrap().owner_authorized);
18842        }
18843
18844        #[tokio::test]
18845        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18846            let mut rig = rig().await;
18847            let mut record = head("s", 1);
18848            record.carriers = vec![carrier(AFT, None)];
18849            let flow_id = "Flow:run-7/step_2!~";
18850            record.attributes.flow_id = Some(flow_id.to_string());
18851            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18852                .await
18853                .unwrap();
18854            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18855                panic!("not a sync reply");
18856            };
18857            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18858            let route = rig
18859                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18860                .await;
18861            assert!(rig.live(&route), "the stamped bind committed");
18862            let stamp = route.stamp().expect("a flow scope carries a stamp");
18863            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18864            assert_eq!(stamp.attributes.agent_id, None);
18865            assert!(!stamp.attributes.delegates);
18866            assert!(stamp.owner_authorized);
18867            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18868            assert_eq!(unscoped.stamp(), None);
18869        }
18870
18871        #[tokio::test]
18872        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18873            let mut rig = rig_with_flow_support(false).await;
18874            let mut record = session(1);
18875            record.attributes.flow_id = Some("flow:7".to_string());
18876            rig.sync(vec![record]).await;
18877            for opener in [OWNER, AFT] {
18878                let body = rig
18879                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18880                    .await;
18881                assert_eq!(body["code"], "target_flow_unsupported");
18882                let message = body["message"].as_str().unwrap();
18883                for required in [PLEXUS, "flow-scopes/v1"] {
18884                    assert!(message.contains(required), "{message}");
18885                }
18886            }
18887        }
18888
18889        #[tokio::test]
18890        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18891            let mut rig = rig_with_flow_support(true).await;
18892            let mut record = session(1);
18893            record.attributes.flow_id = Some("flow:7".to_string());
18894            rig.sync(vec![record]).await;
18895            for opener in [OWNER, AFT] {
18896                let route = rig
18897                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18898                    .await;
18899                assert!(rig.live(&route));
18900                assert_eq!(
18901                    route.stamp().unwrap().attributes.flow_id.as_deref(),
18902                    Some("flow:7")
18903                );
18904            }
18905        }
18906
18907        #[tokio::test]
18908        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18909            use std::future::Future;
18910
18911            let mut rig = rig_with_flow_support(true).await;
18912            let mut record = session(1);
18913            record.attributes.flow_id = Some("flow:7".to_string());
18914            rig.sync(vec![record]).await;
18915            let (client, mut client_rx, frame) =
18916                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18917            // Hold the route-open response permit so admission sees the first
18918            // target but relay reservation cannot capture an endpoint yet.
18919            for _ in 0..64 {
18920                client.egress.try_send(route_bind_ack(1)).unwrap();
18921            }
18922            let handler = rig.handler.clone();
18923            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18924            std::future::poll_fn(|cx| {
18925                assert!(open.as_mut().poll(cx).is_pending());
18926                std::task::Poll::Ready(())
18927            })
18928            .await;
18929            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18930
18931            let old_connection = rig.modules[PLEXUS].0.connection_id;
18932            rig.handler.cleanup_connection(old_connection).unwrap();
18933            let (replacement, mut replacement_rx) = wide_ctx(200);
18934            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18935            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18936            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18937            let hello = Frame::build(
18938                FrameType::Hello,
18939                control_flags(),
18940                0,
18941                0,
18942                200,
18943                serde_json::to_vec(&body).unwrap(),
18944            )
18945            .unwrap();
18946            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18947
18948            client_rx.try_recv().unwrap();
18949            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18950                .await
18951                .expect("the replacement is refused without waiting for a bind ack")
18952                .unwrap();
18953            assert_eq!(replies.len(), 1);
18954            let body = parse_error(&replies[0]);
18955            assert_eq!(body["code"], "target_flow_unsupported");
18956            for required in [PLEXUS, "flow-scopes/v1"] {
18957                assert!(body["message"].as_str().unwrap().contains(required));
18958            }
18959            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18960            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18961            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18962            assert!(rig
18963                .handler
18964                .registry
18965                .get_module_by_connection(replacement.connection_id)
18966                .unwrap()
18967                .is_some());
18968        }
18969
18970        #[tokio::test]
18971        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18972            use std::future::Future;
18973            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18974            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18975            const NEXT_HASH: &str =
18976                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18977            let mut rig = rig().await;
18978            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18979            record.carriers.push(ScopeCarrier::new(Principal::Direct));
18980            rig.sync(vec![record.clone()]).await;
18981            let (client, mut rx, frame) = rig.open_frame(
18982                None,
18983                PLEXUS,
18984                Some(
18985                    rig.selector("s", Some(1))
18986                        .with_bind_proof(Some(PROOF.into())),
18987                ),
18988            );
18989            // Hold egress reservation while the scope owner changes its bind-proof
18990            // hash, so a proof valid at admission must be checked again at relay.
18991            for _ in 0..64 {
18992                client.egress.try_send(route_bind_ack(1)).unwrap();
18993            }
18994            let handler = rig.handler.clone();
18995            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18996            std::future::poll_fn(|cx| {
18997                assert!(open.as_mut().poll(cx).is_pending());
18998                std::task::Poll::Ready(())
18999            })
19000            .await;
19001            record.bind_proof_sha256 = Some(NEXT_HASH.into());
19002            rig.sync(vec![record]).await;
19003            rx.try_recv().unwrap();
19004            let replies = tokio::time::timeout(Duration::from_secs(2), open)
19005                .await
19006                .unwrap()
19007                .unwrap();
19008            assert_eq!(
19009                parse_error(&replies[0])["code"],
19010                error_codes::SCOPE_BIND_PROOF_MISMATCH
19011            );
19012            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
19013            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19014            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19015        }
19016
19017        #[tokio::test]
19018        async fn scope_bind_proof_reserved_openers_ignore_proof() {
19019            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19020            let mut rig = rig().await;
19021            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19022                .await;
19023            for opener in [OWNER, AFT] {
19024                for proof in [None, Some("malformed+proof=")] {
19025                    let route = rig
19026                        .bound(
19027                            Some(opener),
19028                            PLEXUS,
19029                            Some(
19030                                rig.selector("s", Some(1))
19031                                    .with_bind_proof(proof.map(str::to_string)),
19032                            ),
19033                        )
19034                        .await;
19035                    assert!(rig.live(&route));
19036                }
19037            }
19038        }
19039
19040        #[tokio::test]
19041        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
19042            use std::future::Future;
19043            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19044            const NEXT_HASH: &str =
19045                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19046            let mut rig = rig().await;
19047            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19048                .await;
19049            let (client, mut rx, frame) =
19050                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
19051            for _ in 0..64 {
19052                client.egress.try_send(route_bind_ack(1)).unwrap();
19053            }
19054            let handler = rig.handler.clone();
19055            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19056            std::future::poll_fn(|cx| {
19057                assert!(open.as_mut().poll(cx).is_pending());
19058                std::task::Poll::Ready(())
19059            })
19060            .await;
19061            rig.sync(vec![
19062                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
19063            ])
19064            .await;
19065            rx.try_recv().unwrap();
19066            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19067            let bind = std::future::poll_fn(|cx| {
19068                assert!(open.as_mut().poll(cx).is_pending());
19069                module_rx.poll_recv(cx)
19070            })
19071            .await
19072            .unwrap()
19073            .frame;
19074            handler
19075                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
19076                .await
19077                .unwrap();
19078            let replies = tokio::time::timeout(Duration::from_secs(2), open)
19079                .await
19080                .unwrap()
19081                .unwrap();
19082            assert_eq!(replies.len(), 1);
19083            // Proof rechecking allowed the relay. Independent tag settlement
19084            // still asks the opener to retry after any same-epoch content change.
19085            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
19086            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19087            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
19088            let route = rig
19089                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
19090                .await;
19091            assert!(rig.live(&route));
19092        }
19093
19094        #[tokio::test]
19095        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
19096        {
19097            let mut rig = rig_with_flow_support(false).await;
19098            rig.sync(vec![session(1)]).await;
19099            let route = rig
19100                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19101                .await;
19102            assert!(rig.live(&route));
19103            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
19104            let mut record = session(1);
19105            record.attributes.flow_id = Some("flow:7".to_string());
19106            rig.sync(vec![record]).await;
19107            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
19108            assert!(rig.live(&unscoped));
19109            assert_eq!(unscoped.stamp(), None);
19110        }
19111
19112        #[tokio::test]
19113        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
19114            let mut rig = rig().await;
19115            let mut record = session(1);
19116            record.attributes.flow_id = Some("flow:7".to_string());
19117            rig.sync(vec![record.clone()]).await;
19118            let before = rig.forwarding.published_scope_tag(OWNER, "s");
19119            let mut owner_route = rig
19120                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19121                .await;
19122            let mut carrier_route = rig
19123                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19124                .await;
19125            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19126            rig.sync(vec![record.clone()]).await;
19127            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19128            assert!(rig.live(&owner_route) && owner_route.untouched());
19129            assert!(rig.live(&carrier_route) && carrier_route.untouched());
19130
19131            record.attributes.flow_id = Some("flow:8".to_string());
19132            rig.sync(vec![record]).await;
19133            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19134            let before = before.unwrap();
19135            assert_eq!(after.scope_epoch, before.scope_epoch);
19136            assert!(after.version > before.version);
19137            for route in [&mut owner_route, &mut carrier_route] {
19138                assert!(!rig.live(route));
19139                assert_eq!(
19140                    route.closed_reason(),
19141                    RouteCloseReason::ScopeDelegationChanged
19142                );
19143            }
19144            assert!(rig.live(&unscoped) && unscoped.untouched());
19145            // Each provider also receives a GOODBYE for its drained route;
19146            // consume it before expecting the next route.bind on that sink.
19147            for target in [PLEXUS, OTHER] {
19148                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19149                let goodbye = module_rx
19150                    .try_recv()
19151                    .expect("the provider sees the drain")
19152                    .frame;
19153                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19154                assert!(module_rx.try_recv().is_err());
19155            }
19156            let rebound = rig
19157                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19158                .await;
19159            assert_eq!(
19160                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19161                Some("flow:8")
19162            );
19163        }
19164
19165        /// The owner's sync lands between admission and the module's ack. The
19166        /// open is refused by name, the module's other routes stay up, and the
19167        /// reserved pair is released. Changed content is retryable; an ended
19168        /// scope is not.
19169        #[tokio::test]
19170        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19171            let mut rig = rig().await;
19172            rig.sync(vec![session(1)]).await;
19173            let mut cotenant = rig
19174                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19175                .await;
19176
19177            let mut changed = session(1);
19178            changed.child_owners.push(Principal::Reserved {
19179                module_id: MAGIC.to_string(),
19180            });
19181            let mut ended = None;
19182            for (code, next) in [
19183                (error_codes::SCOPE_CHANGED, vec![changed]),
19184                (error_codes::SCOPE_ENDED, Vec::new()),
19185            ] {
19186                let relayed = rig
19187                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19188                    .await;
19189                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19190                ended = Some(next.is_empty());
19191                rig.sync(next).await;
19192                rig.ack(&relayed).await;
19193                let replies = relayed.task.await.unwrap();
19194                assert_eq!(replies.len(), 1, "{replies:?}");
19195                assert_eq!(parse_error(&replies[0])["code"], code);
19196                assert_eq!(
19197                    subc_protocol::error_codes::is_retryable_route_open(code),
19198                    code == error_codes::SCOPE_CHANGED
19199                );
19200                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19201                // The module is told to drop just the binding it created.
19202                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19203                // Collected, because ending the scope also closes the co-tenant
19204                // route, whose GOODBYE comes first.
19205                let mut goodbyes = Vec::new();
19206                while let Ok(outbound) = plexus_rx.try_recv() {
19207                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19208                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19209                }
19210                assert!(
19211                    goodbyes.contains(&(bind_channel, bind_epoch)),
19212                    "{goodbyes:?}"
19213                );
19214                assert!(rig
19215                    .handler
19216                    .registry
19217                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19218                    .unwrap()
19219                    .is_some());
19220            }
19221            assert_eq!(ended, Some(true));
19222            // The co-tenant stayed up through the change, and closed only when
19223            // the scope ended, by the drain rule rather than by the commit.
19224            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19225        }
19226
19227        /// Each row of the drain table on one set of routes: the owner's, a
19228        /// bare carrier's, and a targeted carrier's to each of its targets.
19229        #[tokio::test]
19230        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19231            struct Case {
19232                name: &'static str,
19233                change: fn(&mut ScopeRecord),
19234                /// Closed routes by index: owner->plexus, aft->plexus,
19235                /// broca->plexus, broca->other.
19236                closed: [Option<RouteCloseReason>; 4],
19237            }
19238            use RouteCloseReason::*;
19239            let cases = [
19240                Case {
19241                    name: "a carrier entry removed",
19242                    change: |r| {
19243                        r.carriers.retain(|c| {
19244                            c.principal
19245                                != Principal::Reserved {
19246                                    module_id: AFT.to_string(),
19247                                }
19248                        })
19249                    },
19250                    closed: [None, Some(ScopeCarrierRemoved), None, None],
19251                },
19252                Case {
19253                    name: "a target removed from a carrier",
19254                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19255                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
19256                },
19257                Case {
19258                    name: "a bare carrier narrowed to targets",
19259                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19260                    closed: [None, Some(ScopeCarrierRemoved), None, None],
19261                },
19262                Case {
19263                    name: "delegates turned off",
19264                    change: |r| r.attributes.delegates = false,
19265                    closed: [Some(ScopeDelegationChanged); 4],
19266                },
19267                Case {
19268                    name: "agent_id changed",
19269                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19270                    closed: [Some(ScopeDelegationChanged); 4],
19271                },
19272                Case {
19273                    name: "a carrier added, child owners changed, the record re-sent",
19274                    change: |r| {
19275                        r.carriers.push(carrier(MAGIC, None));
19276                        r.child_owners.push(Principal::Reserved {
19277                            module_id: MAGIC.to_string(),
19278                        });
19279                    },
19280                    closed: [None; 4],
19281                },
19282                Case {
19283                    name: "a target added",
19284                    change: |r| {
19285                        r.carriers[1]
19286                            .targets
19287                            .as_mut()
19288                            .unwrap()
19289                            .push("third".to_string())
19290                    },
19291                    closed: [None; 4],
19292                },
19293                Case {
19294                    name: "delegates turned on",
19295                    change: |r| r.attributes.delegates = true,
19296                    closed: [None; 4],
19297                },
19298            ];
19299            for case in cases {
19300                let mut rig = rig().await;
19301                rig.sync(vec![session(1)]).await;
19302                let scope = || Some(rig_selector("s", Some(1)));
19303                let mut routes = [
19304                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
19305                    rig.bound(Some(AFT), PLEXUS, scope()).await,
19306                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
19307                    rig.bound(Some(BROCA), OTHER, scope()).await,
19308                ];
19309                let mut record = session(1);
19310                (case.change)(&mut record);
19311                rig.sync(vec![record]).await;
19312                for (index, expected) in case.closed.iter().enumerate() {
19313                    let route = &mut routes[index];
19314                    match expected {
19315                        Some(reason) => {
19316                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
19317                            assert_eq!(
19318                                route.closed_reason(),
19319                                *reason,
19320                                "{}: route {index}",
19321                                case.name
19322                            );
19323                        }
19324                        None => {
19325                            assert!(rig.live(route), "{}: route {index} closed", case.name);
19326                            assert!(
19327                                route.untouched(),
19328                                "{}: route {index} was told something",
19329                                case.name
19330                            );
19331                        }
19332                    }
19333                }
19334            }
19335        }
19336
19337        #[tokio::test]
19338        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19339            // Removed, and replaced by a higher epoch.
19340            for next in [Vec::new(), vec![session(2)]] {
19341                let mut rig = rig().await;
19342                rig.sync(vec![session(1)]).await;
19343                let mut route = rig
19344                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19345                    .await;
19346                rig.sync(next).await;
19347                assert!(!rig.live(&route));
19348                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19349            }
19350
19351            // A child whose parent ends: its routes close as parent-ended, the
19352            // child stays live, and routes under the parent close as ended.
19353            let mut rig = rig().await;
19354            let mut child = session(1);
19355            child.scope_ref = "child".to_string();
19356            child.kind = ScopeKind::Worker;
19357            child.parent = Some(ScopeParent::new(
19358                Principal::Reserved {
19359                    module_id: OWNER.to_string(),
19360                },
19361                "s".to_string(),
19362                1,
19363            ));
19364            rig.sync(vec![session(1), child.clone()]).await;
19365            let mut child_route = rig
19366                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19367                .await;
19368            assert_eq!(
19369                child_route.stamp().unwrap().parent_state,
19370                Some(ParentState::Linked)
19371            );
19372            rig.sync(vec![child]).await;
19373            assert!(!rig.live(&child_route));
19374            assert_eq!(
19375                child_route.closed_reason(),
19376                RouteCloseReason::ScopeParentEnded
19377            );
19378        }
19379
19380        #[tokio::test]
19381        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19382        ) {
19383            let mut rig = rig().await;
19384            rig.sync(vec![session(1)]).await;
19385            let mut route = rig
19386                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19387                .await;
19388            let before = rig.forwarding.published_scope_tag(OWNER, "s");
19389            rig.sync(vec![session(1)]).await;
19390            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19391            assert!(rig.live(&route) && route.untouched());
19392
19393            // A call in flight on the route when another carrier is added. A
19394            // forwarded REQUEST holds one credit on the route's flow until the
19395            // module answers; the router takes it exactly like this.
19396            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19397                .forwarding
19398                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19399                .unwrap()
19400            else {
19401                panic!("the route is bound");
19402            };
19403            binding.flow.acquire_tagged(9, false).await.unwrap();
19404            let mut widened = session(1);
19405            widened.carriers.push(carrier(MAGIC, None));
19406            rig.sync(vec![widened]).await;
19407            assert!(rig.live(&route) && route.untouched());
19408            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19409            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19410            // The call's credit is still held on an open flow, so its answer
19411            // will be delivered: closing the route would have closed the flow.
19412            assert_eq!(binding.flow.in_flight(), 1);
19413            binding
19414                .flow
19415                .acquire_tagged(10, false)
19416                .await
19417                .expect("the flow is still open");
19418        }
19419
19420        /// A swap's superseded endpoint keeps its routes until drained; ending
19421        /// the scope closes them there too.
19422        #[tokio::test]
19423        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19424            let mut rig = rig().await;
19425            rig.sync(vec![session(1)]).await;
19426            let mut on_incumbent = rig
19427                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19428                .await;
19429
19430            // Swap plexus: register a candidate and cut over, leaving the
19431            // incumbent superseded with the route still on it.
19432            let (candidate, _candidate_rx) = wide_ctx(9);
19433            let registration = rig
19434                .handler
19435                .registry
19436                .register_candidate_with_control_ops(
19437                    manifest(PLEXUS, PROTOCOL_VERSION),
19438                    PROTOCOL_VERSION,
19439                    candidate.connection_id,
19440                    module_baseline_control_ops(),
19441                )
19442                .unwrap();
19443            rig.forwarding
19444                .register_candidate_module_connection(
19445                    candidate.connection_id,
19446                    PLEXUS.to_string(),
19447                    PROTOCOL_VERSION,
19448                    manifest_concurrency(&registration.manifest),
19449                    candidate.egress.clone(),
19450                )
19451                .unwrap();
19452            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19453            rig.handler
19454                .registry
19455                .promote_candidate(PLEXUS)
19456                .unwrap()
19457                .unwrap();
19458            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19459
19460            rig.sync(Vec::new()).await;
19461            assert!(!rig.live(&on_incumbent));
19462            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19463            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19464            let goodbye = incumbent_rx
19465                .try_recv()
19466                .expect("the superseded endpoint is told")
19467                .frame;
19468            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19469        }
19470    }
19471}
19472
19473#[cfg(test)]
19474mod concurrency_default_exposure_tests {
19475    use super::*;
19476
19477    fn hello_body(role_json: &str) -> Vec<u8> {
19478        format!(
19479            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19480        )
19481        .into_bytes()
19482    }
19483
19484    fn manifest_from(body: &[u8]) -> ModuleManifest {
19485        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19486        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19487            .expect("manifest parses")
19488    }
19489
19490    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19491
19492    #[test]
19493    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19494        let body = hello_body(&format!(
19495            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19496        ));
19497        let manifest = manifest_from(&body);
19498        // Precondition: serde really resolved it to the default, so the typed
19499        // manifest alone cannot answer the question this probe exists for.
19500        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19501        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19502    }
19503
19504    #[test]
19505    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19506        let body = hello_body(&format!(
19507            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19508        ));
19509        let manifest = manifest_from(&body);
19510        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19511        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19512    }
19513
19514    #[test]
19515    fn non_management_roles_are_never_reported() {
19516        let body = hello_body(
19517            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19518        );
19519        let manifest = manifest_from(&body);
19520        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19521    }
19522}