Skip to main content

subc_daemon/
control.rs

1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6    fmt,
7    net::SocketAddr,
8    path::{Path, PathBuf},
9    sync::{Arc, Mutex, RwLock},
10    time::{Duration, Instant as StdInstant},
11};
12
13use serde::{Deserialize, Serialize};
14use subc_control::{
15    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
16    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
17    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
18    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
19    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
20    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
21    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
22};
23use subc_protocol::{
24    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
25    error_codes,
26    manifest::{
27        validate_hello_capability_grammar, validate_hello_confirm_operations,
28        validate_hello_display_name, validate_hello_event_declarations,
29        validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
30        validate_hello_role_declarations, validate_hello_self_signal_declarations,
31        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
32        ProviderRole,
33    },
34    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
35    scope::{
36        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
37        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
38        SCOPE_SYNC_OP,
39    },
40    session::{
41        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
42        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
43        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
44        ROLE_VERSIONS_FIELD,
45    },
46    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
47    Priority, RouteTarget, PROTOCOL_VERSION,
48};
49use tokio::time::{timeout_at, Instant};
50use tracing::{debug, info, instrument::WithSubscriber, warn};
51
52use crate::{
53    caller_identity::CallerIdentity,
54    capability_requirements::{
55        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
56        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
57        RequirementStatus, RuntimeModule,
58    },
59    child_roster::DaemonShutdownFlag,
60    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
61    forwarding::{
62        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
63        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
64        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
65    },
66    observability::{
67        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
68    },
69    privileged_prompt::{prompt_text, PrivilegedOperation},
70    provenance::{
71        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
72    },
73    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
74    router::{daemon_error_flags, RouteCtx, RouterError},
75    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
76    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
77    stderr_tail::{CaptureState, TailEntry},
78    supervise::{
79        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
80        SupervisorHandle, SwapHelloAdmission,
81    },
82    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
83};
84
85/// Lowest envelope version this subc build will negotiate.
86///
87/// Module HELLO negotiation is exact: peers must use the daemon's locked
88/// protocol version. Older and newer peers receive `version_unsupported` and
89/// are not registered.
90pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
91
92const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
93const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
94const CAP_PING_PONG: &str = "ping_pong_v1";
95const CAP_SESSION_ATTACH: &str = "session_attach_v1";
96const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
97
98const SUBC_CONTROL_OPS: &[&str] = &[
99    ops::DAEMON_SHUTDOWN_INTENT,
100    ops::ACTIVITY_RECORD,
101    ops::ACTIVITY_QUERY,
102    ops::ACTIVITY_SUBSCRIBE,
103    ops::SERVER_DESCRIBE,
104    ops::CATALOG_LIST,
105    ops::ROUTE_OPEN,
106    ops::ROUTE_POLL,
107    ops::ROUTE_CLOSING,
108    ops::ROUTE_CLOSED,
109    ops::SUPERVISOR_LIST,
110    ops::SUPERVISOR_RESTART,
111    ops::SUPERVISOR_SWAP,
112    ops::SUPERVISOR_RELOAD,
113    ops::SUPERVISOR_RESCAN,
114    ops::SUPERVISOR_RELEASE_RESERVED,
115    ops::SUPERVISOR_SET_ENABLED,
116    ops::SUPERVISOR_HEALTH_PROBE,
117    ops::SUPERVISOR_HEALTH,
118    ops::SUPERVISOR_STDERR_TAIL,
119    ops::SUPERVISOR_TERMINALS,
120    ops::SUPERVISOR_ROUTES,
121    ops::SUPERVISOR_PROVENANCE,
122    ops::SUPERVISOR_SPAWN_SNAPSHOT,
123    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
124];
125
126/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
127/// announced by a capability in `server.describe` instead, and clients check
128/// that capability before sending it. `ck daemon --json` prints `subc_ops`
129/// verbatim and scripts read that output, so a new name there would change it.
130/// A malformed body for one of these is still refused `invalid_control_body`
131/// rather than as an unknown op.
132const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
133
134mod agent_commands;
135mod rescan_checked;
136use rescan_checked::{RescanMode, RescanPlan};
137
138const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
139    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
140    "supervisor.live_roots",
141    "supervisor.agent_commands",
142    SCOPE_SYNC_OP,
143    SCOPE_APPLY_OP,
144    SCOPE_DESCRIBE_OP,
145    "operator.confirm",
146];
147
148/// Module-originated ops the daemon answers but does not advertise in
149/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
150/// belongs to is incomplete, so no module is told it works before it does.
151const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
152
153const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
154
155/// How long subc waits for a module to ack a relayed route.bind before returning
156/// `module_timeout`. The ack waits on the module's own configure, which for AFT
157/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
158/// and DB-open work — on a cold page cache or a large repo that legitimately
159/// exceeds a couple of seconds. The default is generous because rejecting a VALID
160/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
161/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
162pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
163
164/// How many CONSECUTIVE full-budget relay timeouts against one target module
165/// open that module's bind-relay breaker.
166///
167/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
168/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
169/// so three consecutive full-budget timeouts take ~36s to observe: every client
170/// whose open contributed to opening the breaker had already given up on its
171/// own. That is what makes opening the breaker unable to turn a call that would
172/// have succeeded into a refusal — it can only make an already-failing module
173/// fail faster.
174///
175/// Two would be reachable inside one default deadline. One would convict a
176/// module on a single cold-cache bind, which is exactly the valid-but-slow case
177/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
178pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
179
180/// How long a module's bind-relay breaker stays open before exactly one
181/// `route.open` is let through as a probe.
182///
183/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
184/// re-pays a full-budget stall almost continuously, and the breaker stops being
185/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
186/// deadline: a client that starts retrying after the module recovers has to get
187/// a probe opportunity inside its own deadline, or the breaker converts a
188/// recovered module into a failed call — the failure it exists to prevent,
189/// pointed the other way.
190///
191/// 20s sits between those with room on both sides, and it caps what a wedged
192/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
193/// rather than one per `route.open` per connection. The stall that motivated
194/// this, with its measurements, is written up in
195/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
196/// waited the whole budget out.
197pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
198
199const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
200const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
201
202fn reload_verdict(
203    configured: &Path,
204    spawned_from: Option<&Path>,
205    image: subc_control::RunningImageAgreement,
206) -> PendingReloadVerdict {
207    let path = match spawned_from {
208        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
209            ReloadPathAgreement::Match
210        }
211        Some(spawned_from) => ReloadPathAgreement::Mismatch {
212            configured: configured.to_path_buf(),
213            spawned_from: spawned_from.to_path_buf(),
214        },
215        None => ReloadPathAgreement::Unavailable {
216            reason: if matches!(
217                image,
218                subc_control::RunningImageAgreement::Unavailable {
219                    reason: subc_control::RunningImageUnavailableReason::NotRunning
220                }
221            ) {
222                ReloadPathUnavailableReason::NotRunning
223            } else {
224                ReloadPathUnavailableReason::SpawnedPathUnavailable
225            },
226        },
227    };
228    PendingReloadVerdict {
229        path,
230        image,
231        spec: None,
232    }
233}
234
235fn launch_spec_agreement(
236    configured: &crate::supervise::ModuleSpec,
237    running: bool,
238    spawned: Option<&crate::supervise::LaunchSpecDigest>,
239) -> subc_control::LaunchSpecAgreement {
240    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
241    if !running {
242        return Agreement::Unavailable {
243            reason: Reason::NotRunning,
244        };
245    }
246    match spawned {
247        None => Agreement::Unavailable {
248            reason: Reason::SpawnedSpecUnavailable,
249        },
250        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
251            Agreement::Match
252        }
253        Some(_) => Agreement::Mismatch,
254    }
255}
256
257fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
258    if configured == spawned_from {
259        return true;
260    }
261    #[cfg(windows)]
262    {
263        // Windows spawn records the resolved verbatim path (\\?\...), while
264        // configuration keeps its original spelling. Resolve both with the OS
265        // instead of stripping prefixes or folding case, which could conflate
266        // distinct names in case-sensitive directories. Different resolved
267        // paths still differ even if their files have identical content.
268        match (
269            std::fs::canonicalize(configured),
270            std::fs::canonicalize(spawned_from),
271        ) {
272            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
273            _ => false,
274        }
275    }
276    #[cfg(not(windows))]
277    {
278        false
279    }
280}
281
282#[derive(Clone)]
283struct DaemonProvenanceFacts {
284    build: DaemonBuildProvenance,
285    pid: Option<u32>,
286    started_at_ms: Option<u64>,
287    start_clock: Option<crate::clock::StartClock>,
288    executable_path: Option<PathBuf>,
289    executable_identity: Option<SpawnedFileIdentity>,
290    process_start_time: Option<u64>,
291    probe: ExecutableIdentityProbe,
292}
293
294impl Default for DaemonProvenanceFacts {
295    fn default() -> Self {
296        Self {
297            build: DaemonBuildProvenance {
298                build_git_sha: None,
299                build_lock_digest: None,
300            },
301            pid: None,
302            started_at_ms: None,
303            start_clock: None,
304            executable_path: None,
305            executable_identity: None,
306            process_start_time: None,
307            probe: ExecutableIdentityProbe::default(),
308        }
309    }
310}
311
312#[derive(Debug, Clone)]
313struct SupervisorRescanContext {
314    supervisor: Supervisor,
315    config_path: PathBuf,
316    configured_port: Option<u16>,
317    storage_config: Option<crate::daemon_config::StorageConfig>,
318    admission_facts_carrier_module_id: Option<String>,
319    admission_facts_targets: Option<Vec<String>>,
320    scope_authority_owners: Vec<String>,
321    federated_scope_owner: String,
322    privileged_operator_prompt: PrivilegedOperatorPrompt,
323    /// Counts executed rescans that set out to change the supervised modules'
324    /// staged or applied configuration, counted before the first change so a
325    /// rescan that fails partway still counts. With the daemon incarnation it
326    /// names the state a `supervisor.rescan_checked` preview was computed
327    /// against.
328    generation: Arc<std::sync::atomic::AtomicU64>,
329}
330
331/// Refusal labels passed to `observe_route_open_refusal` that mean the target
332/// module is not serving right now, and so open or extend an outage in the
333/// route outage tracker. Every one of them is only reachable after the target
334/// was found in the registry, which is what keeps an arbitrary client-chosen
335/// id from ever creating tracker state.
336///
337/// Deliberately absent: `not_registered` and `removed` (the id may be
338/// anything a client sent, and a removed module is gone on purpose),
339/// `protocol_none` (such a module never serves routes, so nothing is out),
340/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
341/// capability and admission-facts refusals (they refuse the caller, not a
342/// module outage), and `relay_reservation_failed` (its code ranges over
343/// capacity limits as well as a vanished connection). Capacity, breaker,
344/// relay-timeout and module-rejection refusals do not pass through that
345/// function at all; the breaker logs its own transitions.
346///
347/// The two not-serving refusals that bypass that function record themselves
348/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
349/// `required_capability_unprovided` is not tracked: the module itself is up,
350/// and the outage belongs to the missing provider.
351const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
352    "reloading",
353    "supervisor_not_live",
354    "registration_not_active",
355    "no_forwarding_connection",
356    "relay_send_failed",
357];
358
359/// Real channel-0 control handler for subc itself.
360#[derive(Clone)]
361pub struct ControlHandler {
362    registry: Arc<Registry>,
363    activity: Option<crate::activity::ActivityLedger>,
364    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
365    forwarding: Arc<ForwardingTable>,
366    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
367    supervisor: SupervisorHandle,
368    pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
369    agent_declarations: agent_commands::Queues,
370    list_admission: Arc<tokio::sync::Semaphore>,
371    subc_capabilities: Arc<[String]>,
372    /// Daemon-wide route.bind relay budget. Used as the fallback when the
373    /// target module has no per-module override in
374    /// `route_bind_relay_timeouts`.
375    route_bind_relay_timeout: Duration,
376    /// Per-module route.bind relay budget overrides, keyed by module id. When
377    /// `handle_route_open` resolves the deadline for a target module, a
378    /// per-module entry wins over the daemon-wide value above.
379    route_bind_relay_timeouts: BTreeMap<String, Duration>,
380    /// Per-target-module bind-relay breaker state. Shared with the forwarding
381    /// table, which is where a new module connection resets it.
382    route_bind_breakers: RouteBindBreakers,
383    /// Live relay admissions keyed by target module. Shared through the
384    /// forwarding table so cloned or separately built handlers enforce one cap.
385    route_bind_concurrency: RouteBindConcurrency,
386    /// Start and end of each module's not-serving period as seen by
387    /// `route.open`, so an outage gets one line at each edge instead of only
388    /// the per-refusal INFO lines. Taken from the forwarding table, so every
389    /// handler built over one table shares it.
390    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
391    /// Consecutive relay timeouts that open a module's breaker.
392    route_bind_breaker_threshold: u32,
393    /// How long a breaker stays open before one probe is admitted.
394    route_bind_breaker_cooldown: Duration,
395    health_probe_timeout: Duration,
396    /// Central storage policy. When set, each registering module receives its
397    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
398    storage_config: Option<crate::daemon_config::StorageConfig>,
399    /// The machine id established at boot, served on every HELLO_ACK and on
400    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
401    /// changes the file, never this value. `None` serves no id.
402    machine_id: Option<crate::machine_id::MachineId>,
403    admission_facts_carrier_module_id: Option<String>,
404    admission_facts_targets: Option<Vec<String>>,
405    /// Scope records with their sync authorities and tombstones; see
406    /// `crate::scopes`. Shared by clones of this handler, so every connection
407    /// reads and writes one table.
408    scopes: Arc<RwLock<ScopeTable>>,
409    /// Wakes the expiry loop when an accepted scope change may have added a
410    /// deadline, so the loop can wait without ticking while no live scope has
411    /// one. A permit is stored when nothing waits, so no wake is lost.
412    scope_deadline_added: Arc<tokio::sync::Notify>,
413    /// The configured `scope_authority_owners`, kept so a rescan can report a
414    /// changed value as needing a daemon restart; rescan never applies it.
415    scope_authority_owners: Vec<String>,
416    federated_scope_owner: String,
417    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
418    privileged_operator_prompt: PrivilegedOperatorPrompt,
419    /// The local and peer address of each accepted TCP client connection, read
420    /// from the daemon's own socket, so a privileged request can name its caller.
421    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
422    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
423    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
424    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
425    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
426    pub(crate) operator_apps:
427        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
428    /// One launch entry per app, behind a leaf lock: no other lock is taken
429    /// while it is held. Entries are memory-only, so a daemon or
430    /// installed-service restart drops them and a relaunched daemon never
431    /// mistakes an older process for its own launch.
432    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
433    /// Serializes `app.open` launches: the liveness check in the decision and
434    /// the later publish must not interleave across concurrent calls, or two
435    /// calls could both launch and the second publish would orphan the first
436    /// process. Refusals and `already_running` answers share the same lock;
437    /// `app.open` is rare, so the cost is negligible.
438    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
439    // Portable wire samples exercise dispatch without starting an OS process.
440    #[cfg(test)]
441    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
442    #[cfg(test)]
443    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
444    /// Test-only Windows signature and flash hooks. Production builds carry
445    /// neither: with the publisher unset, the production table never reaches
446    /// verification, and the default attention hook only logs.
447    #[allow(dead_code)]
448    #[cfg(windows)]
449    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
450    #[allow(dead_code)]
451    #[cfg(windows)]
452    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
453    /// Test-only tracking fault hook: makes opening the tracking handle on
454    /// the launched pid fail, so a test can prove an untrackable launch is
455    /// stopped rather than left running.
456    #[allow(dead_code)]
457    #[cfg(windows)]
458    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
459    /// Test-only count of Windows spawn attempts, so a test can prove a
460    /// refused launch started no process. Unit tests only.
461    #[cfg(all(test, windows))]
462    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
463    #[cfg(test)]
464    caller_lookup: Option<TestCallerLookup>,
465    /// The launch nonce each module connection presented at HELLO, which is how
466    /// a `scope.sync` is matched to the owner's current launch.
467    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
468    rescan: Option<SupervisorRescanContext>,
469    connected_clients: ConnectedClients,
470    counters: DaemonCounters,
471    /// Why this daemon serves no local endpoint, when binding one failed. `None`
472    /// when the endpoint is served, and also on a handler that was never told.
473    local_endpoint_unavailable: Option<String>,
474    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
475    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
476    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
477    /// their usual severity (see [`log_requirement_events`]). A handler built
478    /// without one has a flag that is never set.
479    daemon_shutdown: DaemonShutdownFlag,
480    daemon_provenance: DaemonProvenanceFacts,
481    #[cfg(test)]
482    control_dispatch_delay: Option<Duration>,
483    #[cfg(test)]
484    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
485    /// Holds `route.open` after the route.bind is queued to the module and
486    /// before it is marked enqueued, so a test can make the module's reply (or
487    /// a client teardown) land exactly inside that window.
488    #[cfg(test)]
489    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
490    /// Holds a HELLO after its registry entry is reserved and before its
491    /// forwarding endpoint is installed, so a test can read the catalog and
492    /// open routes while the module is registered but not yet routable. A
493    /// blocking barrier because the HELLO handler is synchronous; the test
494    /// runs the HELLO on its own thread.
495    #[cfg(test)]
496    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
497}
498
499impl fmt::Debug for ControlHandler {
500    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
501        f.debug_struct("ControlHandler")
502            .field("registry", &self.registry)
503            .field("forwarding", &self.forwarding)
504            .field("process_liveness", &self.process_liveness.is_some())
505            .field("supervisor", &self.supervisor)
506            .field("subc_capabilities", &self.subc_capabilities)
507            .finish()
508    }
509}
510
511struct RouteOpenRequest {
512    target: RouteTarget,
513    identity: BindIdentity,
514    consumer_identity: Option<ConsumerIdentity>,
515    consumer_capabilities: Option<Vec<String>>,
516    role_versions: Option<BTreeMap<String, String>>,
517    busy_refusals: Option<bool>,
518    admission_facts: Option<serde_json::Value>,
519    scope: Option<ScopeSelector>,
520}
521
522#[derive(Clone, Copy)]
523enum RouteOpenPhase {
524    Admission,
525    ProjectRoot,
526    EgressReservation,
527    BindSend,
528    BindAckWait,
529}
530
531/// Store elapsed time for each route-open phase, including attempts that end
532/// in an early refusal or backend error. Admission also includes rechecks and
533/// commit bookkeeping performed between the other timed phases.
534struct RouteOpenTimings {
535    module_id: String,
536    principal: String,
537    admission: Duration,
538    project_root: Duration,
539    egress_reservation: Duration,
540    bind_send: Duration,
541    bind_ack_wait: Duration,
542    phase: Option<RouteOpenPhase>,
543    phase_started_at: Instant,
544}
545
546impl RouteOpenTimings {
547    fn new() -> Self {
548        Self {
549            module_id: String::new(),
550            // Refusals before attestation must not claim a verified principal.
551            principal: "unresolved".to_string(),
552            admission: Duration::ZERO,
553            project_root: Duration::ZERO,
554            egress_reservation: Duration::ZERO,
555            bind_send: Duration::ZERO,
556            bind_ack_wait: Duration::ZERO,
557            phase: Some(RouteOpenPhase::Admission),
558            phase_started_at: Instant::now(),
559        }
560    }
561
562    fn finish(&mut self) {
563        let elapsed = self.phase_started_at.elapsed();
564        match self.phase.take() {
565            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
566            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
567            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
568            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
569            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
570            None => {}
571        }
572    }
573
574    fn enter(&mut self, phase: RouteOpenPhase) {
575        self.finish();
576        self.phase = Some(phase);
577        self.phase_started_at = Instant::now();
578    }
579}
580
581struct RouteBindReservationGuard {
582    forwarding: Arc<ForwardingTable>,
583    endpoint: ModuleEndpointId,
584    relay_corr: u64,
585    armed: bool,
586}
587
588struct ModuleControlRpcGuard {
589    forwarding: Arc<ForwardingTable>,
590    endpoint: ModuleEndpointId,
591    corr: u64,
592    armed: bool,
593}
594
595impl ModuleControlRpcGuard {
596    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
597        Self {
598            forwarding,
599            endpoint,
600            corr,
601            armed: true,
602        }
603    }
604
605    fn disarm(&mut self) {
606        self.armed = false;
607    }
608}
609
610impl Drop for ModuleControlRpcGuard {
611    fn drop(&mut self) {
612        if self.armed {
613            let _ = self
614                .forwarding
615                .cancel_module_control_rpc(self.endpoint, self.corr);
616        }
617    }
618}
619
620impl RouteBindReservationGuard {
621    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
622        Self {
623            forwarding,
624            endpoint,
625            relay_corr,
626            armed: true,
627        }
628    }
629
630    fn release_and_disarm(&mut self) {
631        if !self.armed {
632            return;
633        }
634        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
635            self.endpoint,
636            self.relay_corr,
637            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
638        ) {
639            send_goodbye_target_best_effort(
640                &self.forwarding.counters(),
641                &target,
642                "canceled route.bind",
643            );
644        }
645        self.armed = false;
646    }
647
648    fn disarm(&mut self) {
649        self.armed = false;
650    }
651}
652
653impl Drop for RouteBindReservationGuard {
654    fn drop(&mut self) {
655        self.release_and_disarm();
656    }
657}
658
659/// Per-target-module circuit breaker around the `route.bind` relay.
660///
661/// The connection reader is serial per connection, so a module whose `on_bind`
662/// sits on the ack blocks every LATER frame on the connections that call it,
663/// including calls to unrelated modules. This does not make any module's bind
664/// fast; it stops the daemon paying the full budget again and again for a
665/// condition it has already observed.
666///
667/// State is keyed by TARGET MODULE and shared by every connection: a wedged
668/// module wedges everyone, so what one connection learned should protect the
669/// rest.
670///
671/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
672/// relay to that module has actually timed out, and is removed again when a
673/// relay is accepted or the module reconnects, so it cannot grow with traffic
674/// or with modules that behave.
675///
676/// # Why a `std` mutex here is not the head-of-line defect again
677///
678/// Acquisition never awaits. The critical section is a hash lookup plus a few
679/// integer updates, with no I/O and no `.await` inside it, so a reader task
680/// cannot be descheduled behind it the way it can behind
681/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
682/// primitive, held for the same kind of work, as the refusal counter this very
683/// path already increments.
684///
685/// It is also NOT on the data-plane splice path: only `route.open` and module
686/// registration touch it, so bound-route frames gain no state check and no
687/// contention.
688#[derive(Debug, Clone, Default)]
689pub(crate) struct RouteBindBreakers {
690    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
691}
692
693#[derive(Debug, Clone, Default)]
694pub(crate) struct RouteBindConcurrency {
695    modules: Arc<Mutex<HashMap<String, usize>>>,
696}
697
698struct RouteBindConcurrencyGuard {
699    concurrency: RouteBindConcurrency,
700    module_id: String,
701}
702
703impl RouteBindConcurrency {
704    /// Admit without waiting. Waiting here would move the bind stall from the
705    /// module reply to a semaphore and restore reader head-of-line blocking.
706    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
707        let mut modules = self
708            .modules
709            .lock()
710            .expect("route.bind concurrency mutex poisoned");
711        let in_flight = modules.entry(module_id.to_string()).or_default();
712        if *in_flight >= limit {
713            return Err(*in_flight);
714        }
715        *in_flight += 1;
716        Ok(RouteBindConcurrencyGuard {
717            concurrency: self.clone(),
718            module_id: module_id.to_string(),
719        })
720    }
721}
722
723impl Drop for RouteBindConcurrencyGuard {
724    fn drop(&mut self) {
725        let mut modules = self
726            .concurrency
727            .modules
728            .lock()
729            .expect("route.bind concurrency mutex poisoned");
730        let remove = {
731            let in_flight = modules
732                .get_mut(&self.module_id)
733                .expect("admitted route.bind has a concurrency entry");
734            *in_flight -= 1;
735            *in_flight == 0
736        };
737        if remove {
738            modules.remove(&self.module_id);
739        }
740    }
741}
742
743#[derive(Debug, Default)]
744struct ModuleBreakerState {
745    /// Relay timeouts observed with no accepted relay in between.
746    consecutive_timeouts: u32,
747    /// `Some` while the breaker is open: the instant the cooldown expires and
748    /// the next arrival may probe. `None` means closed.
749    cooldown_until: Option<Instant>,
750    /// A half-open probe has been admitted and has not settled yet. This is
751    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
752    /// that read the cooldown, so concurrent opens arriving at the moment the
753    /// cooldown expires cannot all decide that they are the probe.
754    probe_in_flight: Option<Arc<()>>,
755}
756
757/// What the breaker decided for one `route.open`, before any relay work.
758enum RouteBindAdmission<'a> {
759    Admitted {
760        guard: RouteBindBreakerGuard<'a>,
761        /// This open is the single half-open probe, so the transition is worth
762        /// one log line.
763        probe: bool,
764    },
765    Refused {
766        consecutive_timeouts: u32,
767        /// What is left of the cooldown. Zero when the refusal is because the
768        /// one probe is already in flight rather than because the cooldown has
769        /// not elapsed.
770        retry_in: Duration,
771        probe_in_flight: bool,
772    },
773}
774
775/// An outstanding admission, which must be told how its relay settled.
776///
777/// `Drop` settles it as inconclusive, so an early return between admission and
778/// the relay -- or the whole handler being cancelled when the client
779/// disconnects -- releases a half-open probe slot instead of leaving the
780/// breaker wedged half-open with no further probes.
781struct RouteBindBreakerGuard<'a> {
782    breakers: RouteBindBreakers,
783    module_id: &'a str,
784    probe_token: Option<Arc<()>>,
785    settled: bool,
786}
787
788impl RouteBindBreakerGuard<'_> {
789    /// The module answered within the budget and took the bind. THE ONLY
790    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
791    /// breaker, which is a transition worth logging.
792    fn record_accepted(&mut self) -> bool {
793        self.settled = true;
794        self.breakers.record_accepted(self.module_id)
795    }
796
797    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
798    /// COUNTS TOWARD OPENING.
799    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
800        self.settled = true;
801        self.breakers.record_timeout(
802            self.module_id,
803            self.probe_token.as_ref(),
804            threshold,
805            cooldown,
806        )
807    }
808
809    /// Everything else: the module REJECTED the bind, its connection went away
810    /// mid-relay, or the waiter was cancelled.
811    ///
812    /// None of these is evidence that a module is slow, and each already has
813    /// its own refusal with its own code. A module that rejects a bind in
814    /// microseconds is healthy and must never be convicted for it; a module
815    /// that died has said nothing about the module that replaces it. So these
816    /// neither increment nor reset the count -- they only release a probe slot.
817    fn record_inconclusive(&mut self) {
818        self.settled = true;
819        self.breakers
820            .record_inconclusive(self.module_id, self.probe_token.as_ref());
821    }
822}
823
824impl Drop for RouteBindBreakerGuard<'_> {
825    fn drop(&mut self) {
826        if !self.settled {
827            self.breakers
828                .record_inconclusive(self.module_id, self.probe_token.as_ref());
829        }
830    }
831}
832
833/// The breaker moved to open, reported so the caller can log it outside the
834/// lock. Opening is rare and load-bearing; the refusals that follow are
835/// frequent and are counted rather than logged.
836struct BreakerOpened {
837    consecutive_timeouts: u32,
838    /// True when a failed probe re-opened an already-open breaker, which reads
839    /// very differently in a log from a first opening.
840    reopened_after_probe: bool,
841}
842
843impl RouteBindBreakers {
844    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
845        self.modules
846            .lock()
847            .expect("route.bind breaker mutex poisoned")
848    }
849
850    /// Decide whether this `route.open` may attempt its relay. Takes the map
851    /// lock and nothing else, and never awaits.
852    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
853        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
854            probe: probe_token.is_some(),
855            guard: RouteBindBreakerGuard {
856                breakers: self.clone(),
857                module_id,
858                probe_token,
859                settled: false,
860            },
861        };
862
863        let mut modules = self.lock();
864        let Some(state) = modules.get_mut(module_id) else {
865            return admitted(None);
866        };
867        let Some(cooldown_until) = state.cooldown_until else {
868            return admitted(None);
869        };
870        if state.probe_in_flight.is_some() {
871            return RouteBindAdmission::Refused {
872                consecutive_timeouts: state.consecutive_timeouts,
873                retry_in: Duration::ZERO,
874                probe_in_flight: true,
875            };
876        }
877        let now = Instant::now();
878        if now < cooldown_until {
879            return RouteBindAdmission::Refused {
880                consecutive_timeouts: state.consecutive_timeouts,
881                retry_in: cooldown_until - now,
882                probe_in_flight: false,
883            };
884        }
885        let token = Arc::new(());
886        state.probe_in_flight = Some(Arc::clone(&token));
887        admitted(Some(token))
888    }
889
890    fn record_accepted(&self, module_id: &str) -> bool {
891        self.lock()
892            .remove(module_id)
893            .is_some_and(|state| state.cooldown_until.is_some())
894    }
895
896    fn record_timeout(
897        &self,
898        module_id: &str,
899        probe_token: Option<&Arc<()>>,
900        threshold: u32,
901        cooldown: Duration,
902    ) -> Option<BreakerOpened> {
903        let mut modules = self.lock();
904        let state = modules.entry(module_id.to_string()).or_default();
905        let was_open = state.cooldown_until.is_some();
906        let was_probe = Self::owns_probe(state, probe_token);
907        if was_probe {
908            state.probe_in_flight = None;
909        }
910        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
911        if state.consecutive_timeouts < threshold {
912            return None;
913        }
914        state.cooldown_until = Some(Instant::now() + cooldown);
915        Some(BreakerOpened {
916            consecutive_timeouts: state.consecutive_timeouts,
917            reopened_after_probe: was_open && was_probe,
918        })
919    }
920
921    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
922        // A relay can finish after the breaker was reset or after it opened
923        // again and started a new probe. Only the guard whose token matches the
924        // active probe may release it, so a late relay never frees a newer probe.
925        state
926            .probe_in_flight
927            .as_ref()
928            .zip(token)
929            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
930    }
931
932    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
933        if let Some(state) = self.lock().get_mut(module_id) {
934            if Self::owns_probe(state, probe_token) {
935                state.probe_in_flight = None;
936            }
937        }
938    }
939
940    /// Discard what was learned about a module, because the process it was
941    /// learned about is gone. Returns the discarded count when it was non-zero.
942    ///
943    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
944    /// `module_id` is a configuration identity that outlives any particular
945    /// child; what the breaker observed was the process behind the module
946    /// connection of the moment. When a new connection registers under that id
947    /// the verdict's subject no longer exists, so the verdict is stale by
948    /// construction rather than merely likely to be wrong. Keeping it would
949    /// apply a dead process's record to a live one, which is the same defect
950    /// class this breaker exists to stop the daemon committing.
951    ///
952    /// A half-open probe in flight is discarded with the rest: it was a
953    /// question about the old process.
954    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
955        self.lock()
956            .remove(module_id)
957            .map(|state| state.consecutive_timeouts)
958            .filter(|discarded| *discarded > 0)
959    }
960
961    /// Open breakers, for the `server.describe` counters object. `None` when
962    /// none is open, so the key stays absent rather than present-and-empty.
963    ///
964    /// This is the operator's answer to "is this module refusing instantly or
965    /// is it fine?", which look identical from a client that retries and then
966    /// succeeds.
967    fn open_snapshot(&self) -> Option<serde_json::Value> {
968        let now = Instant::now();
969        let modules = self.lock();
970        let open = modules
971            .iter()
972            .filter_map(|(module_id, state)| {
973                let cooldown_until = state.cooldown_until?;
974                Some((
975                    module_id.clone(),
976                    serde_json::json!({
977                        "consecutive_timeouts": state.consecutive_timeouts,
978                        "cooldown_remaining_ms":
979                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
980                        "probe_in_flight": state.probe_in_flight.is_some(),
981                    }),
982                ))
983            })
984            .collect::<serde_json::Map<String, serde_json::Value>>();
985        (!open.is_empty()).then_some(serde_json::Value::Object(open))
986    }
987}
988
989impl ControlHandler {
990    pub fn new(registry: Arc<Registry>) -> Self {
991        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
992    }
993
994    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
995        let counters = forwarding.counters();
996        // Taken from the forwarding table rather than created here, so that the
997        // breaker a `route.open` consults is the same one a module's
998        // registration resets, however many handlers are built over one table.
999        let route_bind_breakers = forwarding.route_bind_breakers();
1000        let route_bind_concurrency = forwarding.route_bind_concurrency();
1001        let route_outages = forwarding.route_outages();
1002        Self {
1003            registry,
1004            activity: None,
1005            activity_streams: Arc::new(Mutex::new(HashMap::new())),
1006            forwarding,
1007            process_liveness: None,
1008            supervisor: SupervisorHandle::new(),
1009            resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1010                env!("CARGO_PKG_VERSION").to_owned(),
1011            )),
1012            agent_declarations: agent_commands::Queues::default(),
1013            list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1014            subc_capabilities: Arc::from([
1015                CAP_MANIFEST_REGISTRATION.to_string(),
1016                CAP_CHANNEL_LIFECYCLE.to_string(),
1017                CAP_PING_PONG.to_string(),
1018                CAP_SESSION_ATTACH.to_string(),
1019                CAP_ADMISSION_FACTS_RELAY.to_string(),
1020                CAP_SCOPES_V1.to_string(),
1021                subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1022                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1023                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1024                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1025                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1026                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1027                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1028                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1029                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1030                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1031                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1032                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1033            ]),
1034            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1035            route_bind_relay_timeouts: BTreeMap::new(),
1036            route_bind_breakers,
1037            route_bind_concurrency,
1038            route_outages,
1039            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1040            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1041            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1042            storage_config: None,
1043            machine_id: None,
1044            admission_facts_carrier_module_id: None,
1045            admission_facts_targets: None,
1046            scopes: Arc::new(RwLock::new(ScopeTable::new(
1047                crate::daemon_config::default_scope_authority_owners(),
1048            ))),
1049            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1050            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1051            federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1052            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1053            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1054            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1055            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1056            local_connections: Arc::new(Mutex::new(HashSet::new())),
1057            operator_app_policy: Default::default(),
1058            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1059            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1060            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1061            #[cfg(test)]
1062            app_open_test_platform: None,
1063            #[cfg(test)]
1064            app_open_test_launch: None,
1065            #[cfg(windows)]
1066            windows_verify: None,
1067            #[cfg(windows)]
1068            windows_flash: None,
1069            #[cfg(windows)]
1070            windows_track: None,
1071            #[cfg(all(test, windows))]
1072            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1073            #[cfg(test)]
1074            caller_lookup: None,
1075            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1076            rescan: None,
1077            connected_clients: ConnectedClients::new(),
1078            local_endpoint_unavailable: None,
1079            counters,
1080            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1081            daemon_shutdown: DaemonShutdownFlag::default(),
1082            daemon_provenance: DaemonProvenanceFacts::default(),
1083            #[cfg(test)]
1084            control_dispatch_delay: None,
1085            #[cfg(test)]
1086            provenance_probe_override: None,
1087            #[cfg(test)]
1088            route_bind_relay_sent_hold: None,
1089            #[cfg(test)]
1090            hello_registry_to_forwarding_hold: None,
1091        }
1092    }
1093
1094    /// Set the central storage policy: registering modules then receive their
1095    /// resolved storage descriptor in HELLO_ACK.
1096    pub fn with_storage_config(
1097        mut self,
1098        storage_config: Option<crate::daemon_config::StorageConfig>,
1099    ) -> Self {
1100        self.storage_config = storage_config;
1101        self
1102    }
1103
1104    /// Open the ledger on its own writer thread; an unavailable sink does not
1105    /// prevent the daemon from serving routing or supervisor requests.
1106    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1107        self.activity = Some(crate::activity::ActivityLedger::open(
1108            run_dir.join("activity.jsonl"),
1109            incarnation,
1110        ));
1111        self.supervisor.set_activity(self.activity.clone());
1112        self
1113    }
1114
1115    /// Set the machine id served to every registering module (HELLO_ACK) and on
1116    /// `server.describe`.
1117    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1118        self.machine_id = machine_id;
1119        self
1120    }
1121
1122    /// Configure the exact reserved module and target ids permitted to relay
1123    /// opaque admission facts. Config-file loading validates this authority;
1124    /// this builder keeps the same policy available to embedded test daemons.
1125    pub fn with_admission_facts_config(
1126        mut self,
1127        carrier_module_id: Option<String>,
1128        targets: Option<Vec<String>>,
1129    ) -> Self {
1130        self.admission_facts_carrier_module_id = carrier_module_id;
1131        self.admission_facts_targets = targets;
1132        self
1133    }
1134
1135    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1136    /// Replaces the scope table with an empty one under the new list, so call it
1137    /// while building the handler, before any module can sync.
1138    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1139        self.scopes = Arc::new(RwLock::new(
1140            ScopeTable::new(owners.iter().cloned())
1141                .with_federated_owner(self.federated_scope_owner.clone()),
1142        ));
1143        self.scope_authority_owners = owners;
1144        self
1145    }
1146
1147    /// Set the exclusive federated owner while building the handler, before syncs.
1148    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1149        self.scopes = Arc::new(RwLock::new(
1150            ScopeTable::new(self.scope_authority_owners.iter().cloned())
1151                .with_federated_owner(owner.clone()),
1152        ));
1153        self.federated_scope_owner = owner;
1154        self
1155    }
1156
1157    /// Set the startup policy before installing the rescan context or serving requests.
1158    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1159        self.privileged_operator_prompt = mode;
1160        self
1161    }
1162
1163    /// Override the route.bind relay timeout. Used by tests that assert the
1164    /// timeout path so they don't block on the production-safe default.
1165    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1166        self.route_bind_relay_timeout = timeout;
1167        self
1168    }
1169
1170    /// Install per-module route.bind relay budget overrides. A module id
1171    /// listed here wins over the daemon-wide default set via
1172    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1173    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1174    /// the same `Duration` the bind path will use.
1175    pub fn with_route_bind_relay_timeouts(
1176        mut self,
1177        timeouts: impl IntoIterator<Item = (String, Duration)>,
1178    ) -> Self {
1179        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1180        self
1181    }
1182
1183    /// Resolve the route.bind relay budget for a specific target module id.
1184    /// Per-module overrides win; the daemon-wide value (set via
1185    /// `with_route_bind_relay_timeout` or the built-in default) is the
1186    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1187    /// the same resolution `handle_route_open` will use.
1188    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1189        self.route_bind_relay_timeouts
1190            .get(module_id)
1191            .copied()
1192            .unwrap_or(self.route_bind_relay_timeout)
1193    }
1194
1195    /// Override the per-module bind-relay breaker policy.
1196    ///
1197    /// Used by tests, which cannot spend three production budgets opening a
1198    /// breaker or twenty seconds waiting for its cooldown. The production
1199    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1200    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1201    /// reasoning for the numbers.
1202    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1203        self.route_bind_breaker_threshold = threshold.max(1);
1204        self.route_bind_breaker_cooldown = cooldown;
1205        self
1206    }
1207
1208    #[cfg(test)]
1209    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1210        self.health_probe_timeout = timeout;
1211        self
1212    }
1213
1214    #[cfg(test)]
1215    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1216        self.control_dispatch_delay = Some(delay);
1217        self
1218    }
1219
1220    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1221    /// announce it is parked, once to be released. Build the barrier for two
1222    /// parties, the test being the other one.
1223    #[cfg(test)]
1224    pub(crate) fn with_route_bind_relay_sent_hold(
1225        mut self,
1226        hold: Arc<tokio::sync::Barrier>,
1227    ) -> Self {
1228        self.route_bind_relay_sent_hold = Some(hold);
1229        self
1230    }
1231
1232    /// A HELLO waits on `hold` twice between reserving its registry entry and
1233    /// installing its forwarding endpoint: once to announce it is parked, once
1234    /// to be released. Build the barrier for two parties, the test being the
1235    /// other one, and run the HELLO on a thread of its own.
1236    #[cfg(test)]
1237    pub(crate) fn with_hello_registry_to_forwarding_hold(
1238        mut self,
1239        hold: Arc<std::sync::Barrier>,
1240    ) -> Self {
1241        self.hello_registry_to_forwarding_hold = Some(hold);
1242        self
1243    }
1244
1245    #[cfg(test)]
1246    fn hold_between_registry_and_forwarding(&self) {
1247        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1248            hold.wait();
1249            hold.wait();
1250        }
1251    }
1252
1253    pub fn with_process_liveness(
1254        mut self,
1255        process_liveness: Arc<dyn ModuleProcessLiveness>,
1256    ) -> Self {
1257        self.process_liveness = Some(process_liveness);
1258        self
1259    }
1260
1261    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1262        self.supervisor = supervisor;
1263        self.supervisor.set_activity(self.activity.clone());
1264        self
1265    }
1266
1267    pub fn with_daemon_provenance(
1268        mut self,
1269        pid: u32,
1270        started_at_ms: u64,
1271        executable_path: Option<PathBuf>,
1272        build_git_sha: Option<String>,
1273        build_lock_digest: Option<String>,
1274    ) -> Self {
1275        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1276        let process_start_time = process_start_time(pid);
1277        self.daemon_provenance = DaemonProvenanceFacts {
1278            build: DaemonBuildProvenance {
1279                build_git_sha,
1280                build_lock_digest,
1281            },
1282            pid: Some(pid),
1283            started_at_ms: Some(started_at_ms),
1284            start_clock: None,
1285            executable_path,
1286            executable_identity,
1287            process_start_time,
1288            probe: ExecutableIdentityProbe::default(),
1289        };
1290        self
1291    }
1292
1293    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1294        self.daemon_provenance.start_clock = Some(clock);
1295        self
1296    }
1297
1298    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1299    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1300        self.scopes
1301            .write()
1302            .unwrap_or_else(|poisoned| poisoned.into_inner())
1303            .set_wall_clock(Arc::new(clock));
1304        self
1305    }
1306
1307    #[cfg(test)]
1308    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1309        self.provenance_probe_override = Some(result);
1310        self
1311    }
1312
1313    /// Install the configured module set and its reserved capability bindings.
1314    /// Bindings are configuration-scoped and may point at a provider that has not
1315    /// been installed yet, so this does not require the bound module to exist.
1316    pub fn with_capability_config(
1317        self,
1318        modules: impl IntoIterator<Item = (String, bool)>,
1319        reserved_capabilities: BTreeMap<String, String>,
1320    ) -> Self {
1321        self.capability_evaluator
1322            .configure(modules, reserved_capabilities);
1323        self
1324    }
1325
1326    /// Share the supervisor's daemon-shutdown flag with this handler, so
1327    /// requirement changes after the daemon has begun its announced shutdown
1328    /// are not reported as faults.
1329    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1330        self.daemon_shutdown = flag;
1331        self
1332    }
1333
1334    pub fn with_supervisor_rescan(
1335        mut self,
1336        supervisor: Supervisor,
1337        config_path: impl Into<PathBuf>,
1338        configured_port: Option<u16>,
1339    ) -> Self {
1340        self.rescan = Some(SupervisorRescanContext {
1341            supervisor,
1342            config_path: config_path.into(),
1343            configured_port,
1344            storage_config: self.storage_config.clone(),
1345            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1346            admission_facts_targets: self.admission_facts_targets.clone(),
1347            scope_authority_owners: self.scope_authority_owners.clone(),
1348            federated_scope_owner: self.federated_scope_owner.clone(),
1349            privileged_operator_prompt: self.privileged_operator_prompt,
1350            generation: Arc::default(),
1351        });
1352        self
1353    }
1354
1355    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1356        self.connected_clients = connected_clients;
1357        self
1358    }
1359
1360    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1361    /// an operator can see a bind failure without reading the daemon log.
1362    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1363        self.local_endpoint_unavailable = reason;
1364        self
1365    }
1366
1367    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1368        Arc::clone(&self.forwarding)
1369    }
1370
1371    pub(crate) fn counters(&self) -> DaemonCounters {
1372        self.counters.clone()
1373    }
1374
1375    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1376    /// requirement event without depending on an operator polling a status command.
1377    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1378        tokio::spawn(async move {
1379            loop {
1380                self.capability_evaluator
1381                    .wait_for_change_or_deadline()
1382                    .await;
1383                self.refresh_capability_requirements();
1384            }
1385        });
1386    }
1387
1388    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1389    /// deadline passed during sleep is enforced on the first tick after wake.
1390    /// While no live scope carries a deadline the loop waits for an accepted
1391    /// scope change instead of ticking, so a daemon without deadlines never
1392    /// wakes for expiry.
1393    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1394        tokio::spawn(async move {
1395            loop {
1396                let has_deadlines = self
1397                    .scopes
1398                    .read()
1399                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1400                    .has_deadlines();
1401                if !has_deadlines {
1402                    self.scope_deadline_added.notified().await;
1403                    continue;
1404                }
1405                tokio::time::sleep(Duration::from_secs(1)).await;
1406                if let Err(error) = self.sweep_expired_scopes() {
1407                    warn!(%error, "scope expiry sweep failed");
1408                }
1409            }
1410        });
1411    }
1412
1413    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1414        let mut table = self
1415            .scopes
1416            .write()
1417            .unwrap_or_else(|poisoned| poisoned.into_inner());
1418        let swept = table.sweep_expired();
1419        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1420        drop(table);
1421        self.close_scope_drained_routes(drained);
1422        Ok(())
1423    }
1424
1425    fn publish_scope_changes(
1426        &self,
1427        changes: &[crate::scopes::ScopeTagChange],
1428        expired: &[crate::scopes::ScopeExpired],
1429    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1430        if changes.is_empty() {
1431            return Ok(Vec::new());
1432        }
1433        let drained = self
1434            .forwarding
1435            .publish_scope_changes(changes)
1436            .map_err(RouterError::Forwarding)?;
1437        let mut counts = HashMap::new();
1438        for route in &drained {
1439            *counts
1440                .entry((
1441                    route.scope.owner.as_str(),
1442                    route.scope.scope_ref.as_str(),
1443                    route.scope.tag.scope_epoch,
1444                ))
1445                .or_insert(0usize) += 1;
1446        }
1447        for scope in expired {
1448            let routes_closed = counts
1449                .get(&(
1450                    scope.owner.as_str(),
1451                    scope.scope_ref.as_str(),
1452                    scope.scope_epoch,
1453                ))
1454                .copied()
1455                .unwrap_or(0);
1456            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1457                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1458                routes_closed, "scope expired");
1459        }
1460        Ok(drained)
1461    }
1462
1463    fn runtime_capability_snapshot(
1464        &self,
1465    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1466        let runtime = self
1467            .supervisor
1468            .list()
1469            .into_iter()
1470            .map(|module| {
1471                let status = module.status().map_err(|err| {
1472                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1473                })?;
1474                Ok(RuntimeModule {
1475                    module_id: status.module_id,
1476                    state: status.state,
1477                    enabled: status.enabled,
1478                })
1479            })
1480            .collect::<Result<Vec<_>, RouterError>>()?;
1481        // Include every registration, published or not. The capability
1482        // evaluator counts a module as providing its capabilities as soon as it
1483        // has registered, even before `catalog.list` shows it, so a HELLO in
1484        // progress does not change other modules' required-capability
1485        // verdicts. See `Registry::list_registered_modules`.
1486        let registrations = self.registry.list_registered_modules().map_err(|err| {
1487            RouterError::backend(
1488                0,
1489                0,
1490                format!("failed to list capability registrations: {err}"),
1491            )
1492        })?;
1493        let registrations = registrations
1494            .into_iter()
1495            .map(|registration| RegisteredModule {
1496                module_id: registration.manifest.module_id,
1497                module_version: registration.manifest.module_version,
1498                capabilities: registration.manifest.capabilities,
1499            })
1500            .collect();
1501        Ok((runtime, registrations))
1502    }
1503
1504    /// The capability side effects of a module becoming the active registration
1505    /// for its id: cache its manifest (logging how it differs from the cached
1506    /// one), run the deny census when its declarations call for one, and recompute the
1507    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1508    /// candidate's does not, and the supervisor does it at promotion instead,
1509    /// through [`crate::supervise::SwapPromotionObserver`].
1510    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1511        let cached_registration = RegisteredModule {
1512            module_id: registration.manifest.module_id.clone(),
1513            module_version: registration.manifest.module_version.clone(),
1514            capabilities: registration.manifest.capabilities.clone(),
1515        };
1516        let change = self.capability_evaluator.record_hello(&cached_registration);
1517        log_manifest_change(&cached_registration.module_id, &change);
1518        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1519            self.enforce_capability_denies();
1520        }
1521        self.refresh_capability_requirements();
1522    }
1523
1524    /// Point the shared supervisor handle at this handler for swap promotions.
1525    /// Called wherever a handler is put behind the `Arc` the router serves, so
1526    /// it can be held weakly.
1527    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1528        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1529            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1530        self.supervisor.set_swap_promotion_observer(observer);
1531    }
1532
1533    pub fn refresh_capability_requirements(&self) {
1534        match self.runtime_capability_snapshot() {
1535            Ok((runtime, registrations)) => {
1536                log_requirement_events(
1537                    self.capability_evaluator
1538                        .evaluate_now(&runtime, &registrations),
1539                    &self.daemon_shutdown,
1540                );
1541            }
1542            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1543        }
1544    }
1545
1546    /// Reconcile only live, attested route bindings after a capability deny edge
1547    /// or target claim was added. This is deliberately a control-plane census:
1548    /// the opaque forwarding hot path must not grow a per-frame capability check.
1549    fn enforce_capability_denies(&self) {
1550        let registrations = match self.registry.list_registered_modules() {
1551            Ok(snapshot) => snapshot,
1552            Err(err) => {
1553                warn!(error = %err, "failed to read registrations for capability deny census");
1554                return;
1555            }
1556        };
1557        let manifests = registrations
1558            .into_iter()
1559            .map(|registration| {
1560                (
1561                    registration.manifest.module_id.clone(),
1562                    registration.manifest,
1563                )
1564            })
1565            .collect::<BTreeMap<_, _>>();
1566        let census = match self.forwarding.route_census(None) {
1567            Ok(census) => census,
1568            Err(err) => {
1569                warn!(error = %err, "failed to read route census for capability deny enforcement");
1570                return;
1571            }
1572        };
1573
1574        for (target_module_id, routes) in census {
1575            let Some(target_manifest) = manifests.get(&target_module_id) else {
1576                continue;
1577            };
1578            let mut closed_routes = Vec::new();
1579            let mut goodbyes = Vec::new();
1580            for route in routes {
1581                let Principal::Reserved {
1582                    module_id: opening_module_id,
1583                } = &route.principal
1584                else {
1585                    continue;
1586                };
1587                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1588                    continue;
1589                };
1590                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1591                    continue;
1592                };
1593
1594                match self.forwarding.release_client_route(
1595                    route.goodbye_target.connection_id,
1596                    route.goodbye_target.channel,
1597                    route.goodbye_target.epoch,
1598                ) {
1599                    Ok(RouteRelease::Removed(module_goodbye)) => {
1600                        warn!(
1601                            opening_module_id,
1602                            target_module_id,
1603                            capability,
1604                            "force-closing route because an attested capability deny edge now matches"
1605                        );
1606                        goodbyes.push(module_goodbye);
1607                        goodbyes.push(route.goodbye_target.clone());
1608                        closed_routes.push(route);
1609                    }
1610                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1611                    Err(err) => warn!(
1612                        opening_module_id,
1613                        target_module_id,
1614                        capability,
1615                        error = %err,
1616                        "failed to force-close capability-denied route"
1617                    ),
1618                }
1619            }
1620
1621            if closed_routes.is_empty() {
1622                continue;
1623            }
1624            send_route_control_pushes(
1625                &self.forwarding,
1626                closed_routes,
1627                ClientControlPush::RouteClosed {
1628                    module_id: target_module_id,
1629                    channels: Vec::new(),
1630                    reason: RouteCloseReason::CapabilityDenied,
1631                    drained: false,
1632                    abandoned: 0,
1633                    excluded_subscriptions: 0,
1634                    terminal: Some(false),
1635                },
1636            );
1637            // route.closed records the reason; the route GOODBYE ends the
1638            // client's handle and settles any calls still pending on it.
1639            self.emit_route_goodbyes(goodbyes);
1640        }
1641    }
1642
1643    /// Why a registered module is not accepting new route binds, or `None` when
1644    /// it is. This is the module's effective readiness: its declared readiness
1645    /// first, then every `need: required` capability it declares evaluating to
1646    /// `provided`. `route.open` and `catalog.list` both read it here so the
1647    /// catalog never reports a module routable that `route.open` would refuse.
1648    fn not_ready_reason(
1649        &self,
1650        registration: &crate::registry::ModuleRegistration,
1651    ) -> Option<NotReadyReason> {
1652        if !registration.ready {
1653            return Some(NotReadyReason {
1654                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1655                capability: None,
1656            });
1657        }
1658        self.first_unprovided_required_capability(registration)
1659            .map(|capability| NotReadyReason {
1660                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1661                capability: Some(capability),
1662            })
1663    }
1664
1665    /// The lexicographically first capability this registration declares
1666    /// `need: required` whose evaluator verdict is not `provided`.
1667    ///
1668    /// The verdicts are the capability evaluator's own; nothing here decides
1669    /// what "provided" means. The evaluator counts a capability provided as
1670    /// soon as a module claiming it has REGISTERED, not once that module is
1671    /// ready. That distinction is what keeps two modules that require each
1672    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1673    /// is ready", each would wait for the other to become ready first and
1674    /// neither ever would. Do not tighten it to readiness.
1675    ///
1676    /// A required capability with no verdict at all means this registration's
1677    /// HELLO or catalog.update landed after the last recompute; recompute once
1678    /// rather than let a missing verdict read as either answer. If it is still
1679    /// missing (the recompute itself failed) the capability counts as
1680    /// unprovided: the refusal is retryable, and routing a module whose
1681    /// required provider is unknown is the outcome this check exists to stop.
1682    fn first_unprovided_required_capability(
1683        &self,
1684        registration: &crate::registry::ModuleRegistration,
1685    ) -> Option<String> {
1686        let required = registration
1687            .manifest
1688            .capabilities
1689            .iter()
1690            .flat_map(|declarations| declarations.requires.iter())
1691            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1692            .map(|requirement| requirement.capability.as_str())
1693            .collect::<BTreeSet<_>>();
1694        if required.is_empty() {
1695            return None;
1696        }
1697        let module_id = registration.manifest.module_id.as_str();
1698        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1699        if required
1700            .iter()
1701            .any(|capability| verdict(capability).is_none())
1702        {
1703            self.refresh_capability_requirements();
1704        }
1705        required
1706            .into_iter()
1707            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1708            .map(str::to_string)
1709    }
1710
1711    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1712        self.capability_evaluator
1713            .statuses()
1714            .into_iter()
1715            .map(capability_requirement_status)
1716            .collect()
1717    }
1718
1719    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1720    /// registration-release watch. The signal is what the supervisor waits on
1721    /// before spawning a replacement, so it must only fire once forwarding
1722    /// teardown is also done (see [`Self::cleanup_connection`] /
1723    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1724    /// admitted but its forwarding endpoint could not be installed.
1725    fn deregister_connection(
1726        &self,
1727        connection_id: ConnectionId,
1728        reason: RegistrationEndReason,
1729    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1730        self.registry
1731            .deregister_connection_with_reason(connection_id, reason)
1732    }
1733
1734    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1735        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1736            return None;
1737        }
1738        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1739            parse_client_control_request(&frame.body)
1740        else {
1741            return None;
1742        };
1743        Some(target_module_id(&target).to_string())
1744    }
1745
1746    pub(crate) fn route_open_capacity_refusal(
1747        &self,
1748        ctx: &RouteCtx,
1749        frame: &Frame,
1750        target_module_id: &str,
1751        in_flight: usize,
1752        limit: usize,
1753    ) -> Result<Frame, RouterError> {
1754        // Sent before the request reaches `handle_route_open`, so the caller
1755        // fields come from the raw frame here.
1756        let caller = self.route_open_caller_from_frame(frame);
1757        self.route_open_admission_refusal_frame(
1758            ctx,
1759            &caller,
1760            frame,
1761            target_module_id,
1762            "open_admission_full",
1763            (in_flight, limit),
1764            format!(
1765                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1766            ),
1767        )
1768    }
1769
1770    fn route_open_target_capacity_refusal(
1771        &self,
1772        ctx: &RouteCtx,
1773        caller: &RouteOpenCaller,
1774        frame: &Frame,
1775        target_module_id: &str,
1776        in_flight: usize,
1777    ) -> Result<Frame, RouterError> {
1778        self.route_open_admission_refusal_frame(
1779            ctx,
1780            caller,
1781            frame,
1782            target_module_id,
1783            "target_binds_full",
1784            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1785            format!(
1786                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1787            ),
1788        )
1789    }
1790
1791    /// Admission pressure clears as existing binds settle, so its refusal must
1792    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1793    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1794    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1795    /// cannot currently reach its target; `module_timeout` would falsely claim
1796    /// that a wait expired. A new, cleaner code would be terminal to deployed
1797    /// clients, so it requires a client-tolerance rollout before daemon emission.
1798    #[allow(clippy::too_many_arguments)]
1799    fn route_open_admission_refusal_frame(
1800        &self,
1801        ctx: &RouteCtx,
1802        caller: &RouteOpenCaller,
1803        frame: &Frame,
1804        target_module_id: &str,
1805        reason: &'static str,
1806        (in_flight, limit): (usize, usize),
1807        message: impl Into<String>,
1808    ) -> Result<Frame, RouterError> {
1809        let code = error_codes::TARGET_UNAVAILABLE;
1810        self.counters.increment_route_open_refused(code);
1811        info!(
1812            target: "control",
1813            code,
1814            reason,
1815            module_id = ?target_module_id,
1816            connection_id = ctx.connection_id.get(),
1817            principal = %caller.principal,
1818            scope = %caller.scope,
1819            harness = %caller.harness,
1820            in_flight,
1821            limit,
1822            "route.open refused"
1823        );
1824        control_error_frame(frame, code, message.into())
1825    }
1826
1827    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1828    ///
1829    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1830    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1831    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1832    #[cfg(test)]
1833    pub fn handle_control(
1834        &self,
1835        connection_id: ConnectionId,
1836        frame: Frame,
1837    ) -> Result<Vec<Frame>, RouterError> {
1838        match frame.header.ty {
1839            FrameType::Ping => Ok(vec![pong(&frame)?]),
1840            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1841            FrameType::Goodbye => self.handle_goodbye(connection_id),
1842            ty => Ok(vec![control_error_frame(
1843                &frame,
1844                "unsupported_control_frame",
1845                format!("unsupported channel-0 frame {ty:?}"),
1846            )?]),
1847        }
1848    }
1849
1850    pub async fn handle_control_frame(
1851        &self,
1852        ctx: &RouteCtx,
1853        frame: Frame,
1854    ) -> Result<Vec<Frame>, RouterError> {
1855        self.handle_control_frame_timed(ctx, frame, None).await
1856    }
1857
1858    #[cfg(any(test, feature = "test-support"))]
1859    pub fn with_test_operator_app_policy(
1860        mut self,
1861        policy: crate::operator_apps::TestOperatorAppPolicy,
1862    ) -> Self {
1863        self.operator_app_policy = policy.into();
1864        self
1865    }
1866
1867    /// Inject the Windows signature check for tests. The injected check
1868    /// answers the verified version string or the rejection reason; the
1869    /// daemon enforces the floor itself on the returned version.
1870    #[cfg(all(windows, any(test, feature = "test-support")))]
1871    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1872        self.windows_verify = Some(verify);
1873        self
1874    }
1875
1876    /// Record which pids the daemon would flash for `already_running`.
1877    #[cfg(all(windows, any(test, feature = "test-support")))]
1878    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1879        self.windows_flash = Some(flash);
1880        self
1881    }
1882
1883    /// Inject a tracking failure for tests: makes `track_launched_process`
1884    /// fail so a test can prove a launch that cannot be tracked is stopped,
1885    /// never left running.
1886    #[cfg(all(windows, any(test, feature = "test-support")))]
1887    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1888        self.windows_track = Some(track);
1889        self
1890    }
1891
1892    pub(crate) fn record_captured_peer(
1893        &self,
1894        connection: ConnectionId,
1895        peer: subc_os::local_peer::CapturedPeer,
1896    ) {
1897        self.record_observed_peer(connection, peer.observation().clone());
1898        #[cfg(windows)]
1899        if let Some(stamp) = self.stamp_windows_launch(&peer) {
1900            self.operator_apps
1901                .lock()
1902                .unwrap_or_else(|p| p.into_inner())
1903                .insert(connection, stamp);
1904        }
1905        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1906            self.operator_apps
1907                .lock()
1908                .unwrap_or_else(|p| p.into_inner())
1909                .insert(connection, stamp);
1910        }
1911    }
1912
1913    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1914        self.local_connections
1915            .lock()
1916            .unwrap_or_else(|p| p.into_inner())
1917            .insert(connection);
1918    }
1919
1920    // Keep the connection-level delivery decision shared with route.bind. The
1921    // target's capability opt-in is separate from recognition of the caller.
1922    fn connection_operator_app(
1923        &self,
1924        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1925    ) -> Option<subc_protocol::peer::OperatorApp> {
1926        stamp
1927            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1928            .map(|stamp| stamp.app.clone())
1929    }
1930
1931    fn handle_connection_describe(
1932        &self,
1933        ctx: &RouteCtx,
1934        frame: Frame,
1935    ) -> Result<Vec<Frame>, RouterError> {
1936        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1937            return Ok(vec![control_error_frame(
1938                &frame,
1939                "invalid_control_body",
1940                "malformed connection.describe body",
1941            )?]);
1942        }
1943        let local = self
1944            .local_connections
1945            .lock()
1946            .unwrap_or_else(|p| p.into_inner())
1947            .contains(&ctx.connection_id);
1948        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1949        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1950        Ok(vec![control_response_body_frame(
1951            &frame,
1952            &subc_control::ConnectionDescribeReply::new(local, operator_app),
1953            ops::CONNECTION_DESCRIBE,
1954        )?])
1955    }
1956
1957    pub(crate) fn record_observed_peer(
1958        &self,
1959        connection: ConnectionId,
1960        peer: subc_os::local_peer::ObservedPeer,
1961    ) {
1962        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1963        let peer = match peer {
1964            OsPeer::Unobserved => ObservedPeer::unobserved(),
1965            OsPeer::Observed(peer) => {
1966                let observed = match peer.user {
1967                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1968                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1969                };
1970                observed
1971                    .with_exe(
1972                        peer.exe
1973                            .and_then(|path| path.into_os_string().into_string().ok()),
1974                    )
1975                    .with_signing_id(peer.signing_id)
1976            }
1977        };
1978        self.observed_peers
1979            .lock()
1980            .unwrap_or_else(|p| p.into_inner())
1981            .insert(connection, peer);
1982    }
1983
1984    pub(crate) fn record_caller_endpoints(
1985        &self,
1986        connection: ConnectionId,
1987        endpoints: Option<(SocketAddr, SocketAddr)>,
1988    ) {
1989        let mut callers = self
1990            .caller_endpoints
1991            .lock()
1992            .unwrap_or_else(|p| p.into_inner());
1993        callers.remove(&connection);
1994        if let Some(endpoints) = endpoints {
1995            callers.insert(connection, endpoints);
1996        }
1997    }
1998
1999    #[cfg(test)]
2000    pub(crate) fn with_caller_lookup_for_test(
2001        mut self,
2002        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2003    ) -> Self {
2004        self.caller_lookup = Some(Arc::new(lookup));
2005        self
2006    }
2007
2008    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2009        let endpoints = self
2010            .caller_endpoints
2011            .lock()
2012            .unwrap_or_else(|p| p.into_inner())
2013            .get(&connection)
2014            .copied();
2015        #[cfg(test)]
2016        let lookup = self.caller_lookup.clone();
2017        tokio::task::spawn_blocking(move || {
2018            #[cfg(test)]
2019            if let Some(lookup) = lookup {
2020                return lookup(endpoints);
2021            }
2022            endpoints
2023                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2024                .unwrap_or_default()
2025        })
2026        .await
2027        .unwrap_or_default()
2028    }
2029
2030    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2031        frame.header.channel == 0
2032            && frame.header.ty == FrameType::Request
2033            && self
2034                .forwarding
2035                .module_endpoint_for_connection(ctx.connection_id)
2036                .is_ok_and(|endpoint| endpoint.is_none())
2037            && parse_client_control_request(&frame.body)
2038                .ok()
2039                .is_some_and(|request| privileged_operation(&request).is_some())
2040    }
2041
2042    pub(crate) async fn handle_control_frame_timed(
2043        &self,
2044        ctx: &RouteCtx,
2045        frame: Frame,
2046        dispatch_started_at: Option<StdInstant>,
2047    ) -> Result<Vec<Frame>, RouterError> {
2048        match frame.header.ty {
2049            FrameType::Ping => Ok(vec![pong(&frame)?]),
2050            FrameType::Hello => {
2051                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2052            }
2053            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2054            FrameType::Cancel => {
2055                // Channel-0 cancellation also fences pending declarations;
2056                // successful cancellation has no ordinary reply.
2057                if self
2058                    .agent_declarations
2059                    .cancel(ctx.connection_id, frame.header.corr)
2060                    || self
2061                        .forwarding
2062                        .operator_confirms()
2063                        .cancel(ctx.connection_id, frame.header.corr)
2064                    || self
2065                        .supervisor
2066                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2067                    || self.cancel_activity_subscription(ctx, &frame)?
2068                {
2069                    Ok(Vec::new())
2070                } else {
2071                    Ok(vec![control_error_frame(
2072                        &frame,
2073                        "unknown_subscription",
2074                        "no subscription has this correlation id",
2075                    )?])
2076                }
2077            }
2078            FrameType::Request => {
2079                // This additive operation is not part of the existing exhaustive
2080                // module-control enum. Probe the op before decoding that enum.
2081                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2082                if op
2083                    .as_ref()
2084                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2085                {
2086                    return self.handle_activity_record(ctx, frame).await;
2087                }
2088                if op.as_ref().is_some_and(|probe| {
2089                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2090                }) {
2091                    return self.handle_activity_read(ctx, frame).await;
2092                }
2093                if op
2094                    .as_ref()
2095                    .is_some_and(|probe| probe.op == "operator.confirm")
2096                {
2097                    return self.handle_operator_confirm(ctx, frame);
2098                }
2099                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2100                    return self
2101                        .observe_control_wait(
2102                            ctx,
2103                            None,
2104                            ops::APP_OPEN,
2105                            self.handle_app_open(ctx, frame),
2106                        )
2107                        .await;
2108                }
2109                if op
2110                    .as_ref()
2111                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2112                {
2113                    return self.handle_connection_describe(ctx, frame);
2114                }
2115                if op
2116                    .as_ref()
2117                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2118                {
2119                    return self.handle_scope_served_by(ctx.connection_id, frame);
2120                }
2121                if self
2122                    .forwarding
2123                    .module_endpoint_for_connection(ctx.connection_id)
2124                    .map_err(RouterError::Forwarding)?
2125                    .is_some()
2126                {
2127                    if !is_known_module_request_op(&frame.body) {
2128                        return Ok(vec![control_error_frame(
2129                            &frame,
2130                            "unsupported_control_frame",
2131                            "module-originated channel-0 REQUEST is not supported",
2132                        )?]);
2133                    }
2134                    let request = match parse_module_control_request_from_module(&frame.body) {
2135                        Ok(request) => request,
2136                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2137                            return Ok(vec![control_error_frame(
2138                                &frame,
2139                                "unsupported_control_frame",
2140                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2141                            )?])
2142                        }
2143                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2144                            return Ok(vec![control_error_frame(
2145                                &frame,
2146                                "invalid_control_body",
2147                                format!("malformed module control body: {err}"),
2148                            )?])
2149                        }
2150                    };
2151                    let op = module_control_request_op(&request);
2152                    let corr = frame.header.corr;
2153                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2154                    if let ModuleControlRequestFromModule::AgentCommands {
2155                        started,
2156                        finished,
2157                        known,
2158                    } = request
2159                    {
2160                        return self
2161                            .agent_declarations
2162                            .enqueue(
2163                                self,
2164                                ctx,
2165                                frame,
2166                                crate::resource_totals::Declaration {
2167                                    started,
2168                                    finished,
2169                                    known,
2170                                },
2171                            )
2172                            .await;
2173                    }
2174                    let result =
2175                        self.handle_module_control_request(ctx.connection_id, frame, request);
2176                    log_slow_control_dispatch(
2177                        dispatch_started_at,
2178                        op,
2179                        ctx.connection_id,
2180                        corr,
2181                        None,
2182                    );
2183                    return result;
2184                }
2185
2186                if is_known_module_request_op(&frame.body) {
2187                    let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2188                        .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2189                    return Ok(vec![control_error_frame(
2190                        &frame,
2191                        if is_agents { "not_supervised" } else { "not_registered" },
2192                        "module-originated control requests require the module's own registered connection",
2193                    )?]);
2194                }
2195
2196                let request = match parse_client_control_request(&frame.body) {
2197                    Ok(request) => request,
2198                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2199                        return Ok(vec![control_error_frame(
2200                            &frame,
2201                            "unknown_control_op",
2202                            format!("unknown client control op: {err}"),
2203                        )?])
2204                    }
2205                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2206                        return Ok(vec![control_error_frame(
2207                            &frame,
2208                            "invalid_control_body",
2209                            format!("malformed client control body: {err}"),
2210                        )?])
2211                    }
2212                };
2213                let op = client_control_request_op(&request);
2214                let corr = frame.header.corr;
2215                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2216                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2217                #[cfg(test)]
2218                if let Some(delay) = self.control_dispatch_delay {
2219                    tokio::time::sleep(delay).await;
2220                }
2221                let result = self
2222                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2223                    .await;
2224                if let Some(timings) = route_open_timings.as_mut() {
2225                    timings.finish();
2226                }
2227                log_slow_control_dispatch(
2228                    dispatch_started_at,
2229                    op,
2230                    ctx.connection_id,
2231                    corr,
2232                    route_open_timings.as_ref(),
2233                );
2234                result
2235            }
2236            FrameType::Push => {
2237                let Some(endpoint) = self
2238                    .forwarding
2239                    .module_endpoint_for_connection(ctx.connection_id)
2240                    .map_err(RouterError::Forwarding)?
2241                else {
2242                    return Ok(vec![control_error_frame(
2243                        &frame,
2244                        "unsupported_control_frame",
2245                        "client-originated channel-0 PUSH is not supported",
2246                    )?]);
2247                };
2248                self.handle_status_update(endpoint, frame)
2249            }
2250            FrameType::Response | FrameType::Error
2251                if self
2252                    .forwarding
2253                    .module_endpoint_for_connection(ctx.connection_id)
2254                    .map_err(RouterError::Forwarding)?
2255                    .is_some() =>
2256            {
2257                self.handle_module_relay_response(ctx.connection_id, frame)
2258            }
2259            ty => Ok(vec![control_error_frame(
2260                &frame,
2261                "unsupported_control_frame",
2262                format!("unsupported channel-0 frame {ty:?}"),
2263            )?]),
2264        }
2265    }
2266
2267    pub fn cleanup_connection(
2268        &self,
2269        connection_id: ConnectionId,
2270    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2271        self.local_connections
2272            .lock()
2273            .unwrap_or_else(|p| p.into_inner())
2274            .remove(&connection_id);
2275        self.caller_endpoints
2276            .lock()
2277            .unwrap_or_else(|p| p.into_inner())
2278            .remove(&connection_id);
2279        self.build_class_warned
2280            .lock()
2281            .unwrap_or_else(|p| p.into_inner())
2282            .remove(&connection_id);
2283        self.observed_peers
2284            .lock()
2285            .unwrap_or_else(|p| p.into_inner())
2286            .remove(&connection_id);
2287        self.operator_apps
2288            .lock()
2289            .unwrap_or_else(|p| p.into_inner())
2290            .remove(&connection_id);
2291        self.forwarding
2292            .operator_confirms()
2293            .connection_closed(connection_id);
2294        self.cleanup_connection_with_end_reason(
2295            connection_id,
2296            RegistrationEndReason::ConnectionClosed,
2297        )
2298    }
2299
2300    fn cleanup_connection_with_end_reason(
2301        &self,
2302        connection_id: ConnectionId,
2303        requested_reason: RegistrationEndReason,
2304    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2305        self.agent_declarations.close(connection_id);
2306        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2307            self.registry
2308                .get_module_by_connection(connection_id)?
2309                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2310                .and_then(|module| module.registration_end_reason().ok().flatten())
2311                .unwrap_or(requested_reason)
2312        } else {
2313            requested_reason
2314        };
2315        let crash_closed = self
2316            .registry
2317            .get_module_by_connection(connection_id)?
2318            .and_then(|registration| {
2319                self.forwarding
2320                    .module_endpoint_for_connection(connection_id)
2321                    .ok()
2322                    .flatten()
2323                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2324                    .map(|routes| (registration.manifest.module_id, routes))
2325            });
2326        let crash_closed = crash_closed.map(|(module_id, routes)| {
2327            let terminal = match self.supervisor.get(&module_id) {
2328                None => false,
2329                Some(module) => match module.will_recover_after_connection_loss() {
2330                    Ok(will_recover) => !will_recover,
2331                    Err(err) => {
2332                        warn!(
2333                            %module_id,
2334                            error = %err,
2335                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2336                        );
2337                        false
2338                    }
2339                },
2340            };
2341            // The forwarding table gates all providers at the start of daemon
2342            // shutdown, before their connections are closed. An ordinary
2343            // module disconnect still reports crash if that gate is not set.
2344            let (reason, drained) = match self
2345                .forwarding
2346                .daemon_drain_result_for_connection(connection_id)
2347            {
2348                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2349                Ok(None) => (RouteCloseReason::Crash, false),
2350                Err(err) => {
2351                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2352                    (RouteCloseReason::Crash, false)
2353                }
2354            };
2355            (module_id, routes, reason, drained, terminal)
2356        });
2357        let registrations = self.deregister_connection(connection_id, end_reason);
2358        let cleanup = if crash_closed.is_some() {
2359            self.forwarding.cleanup_connection_counted(connection_id)
2360        } else {
2361            // A module connection's teardown needs the count of abandoned
2362            // route.bind relays for its route.closed notice below. Any other
2363            // connection, such as a client's, sends no such notice and needs
2364            // only its routes released, so it uses the route-only wrapper and
2365            // reports zero.
2366            self.forwarding
2367                .cleanup_connection(connection_id)
2368                .map(|released| crate::forwarding::ConnectionCleanup {
2369                    released,
2370                    abandoned_relays: 0,
2371                })
2372        };
2373        // The route.closed push waits for forwarding teardown because only
2374        // teardown knows how many pending route.bind relays it aborted. It still
2375        // goes out before the GOODBYEs for the released routes, and its targets
2376        // were captured above, before teardown removed those routes.
2377        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2378            let abandoned = cleanup
2379                .as_ref()
2380                .map_or(0, |cleanup| cleanup.abandoned_relays);
2381            send_route_control_pushes(
2382                &self.forwarding,
2383                routes,
2384                ClientControlPush::RouteClosed {
2385                    module_id,
2386                    channels: Vec::new(),
2387                    reason,
2388                    drained,
2389                    abandoned,
2390                    excluded_subscriptions: 0,
2391                    terminal: Some(terminal),
2392                },
2393            );
2394        }
2395        if let Ok(cleanup) = cleanup {
2396            self.emit_route_goodbyes(cleanup.released);
2397        }
2398        // Signal the registration-release watch only now that BOTH registry and
2399        // forwarding teardown are done, so a supervisor waiting to spawn a
2400        // replacement never observes release while old routes still exist.
2401        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2402            crate::supervise::notify_registration_release();
2403            self.capability_evaluator.wake_deadline_loop();
2404            self.refresh_capability_requirements();
2405        }
2406        self.supervisor.remove_spawn_subscribers(connection_id);
2407        self.activity_streams
2408            .lock()
2409            .unwrap_or_else(|p| p.into_inner())
2410            .retain(|(id, _), task| {
2411                if *id == connection_id {
2412                    task.abort();
2413                    false
2414                } else {
2415                    true
2416                }
2417            });
2418        // Sync authority dies with its connection, so the owner's next
2419        // connection can take it; the owner's scopes stay as they are.
2420        self.hello_launch_nonces
2421            .lock()
2422            .unwrap_or_else(|poisoned| poisoned.into_inner())
2423            .forget(connection_id);
2424        self.scopes
2425            .write()
2426            .unwrap_or_else(|poisoned| poisoned.into_inner())
2427            .release_connection(connection_id);
2428        registrations
2429    }
2430
2431    pub(crate) fn handle_route_goodbye(
2432        &self,
2433        connection_id: ConnectionId,
2434        route_channel: u16,
2435        route_epoch: u32,
2436    ) -> Result<bool, RouterError> {
2437        debug!(
2438            connection_id = connection_id.get(),
2439            route_channel, route_epoch, "handling route GOODBYE"
2440        );
2441        let RouteRelease::Removed(released_route) = self
2442            .forwarding
2443            .release_client_route(connection_id, route_channel, route_epoch)
2444            .map_err(RouterError::Forwarding)?
2445        else {
2446            return Ok(false);
2447        };
2448        self.emit_route_goodbyes(vec![released_route]);
2449        Ok(true)
2450    }
2451
2452    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2453        for released in released_routes {
2454            let frame = match Frame::build_with_version(
2455                released.negotiated_ver,
2456                FrameType::Goodbye,
2457                control_flags(),
2458                released.channel,
2459                released.epoch,
2460                0,
2461                Vec::new(),
2462            ) {
2463                Ok(frame) => frame,
2464                Err(err) => {
2465                    warn!(
2466                        route_channel = released.channel,
2467                        error = %err,
2468                        "failed to build route GOODBYE frame"
2469                    );
2470                    continue;
2471                }
2472            };
2473            if !released.close_on_delivery_failure() {
2474                crate::forwarding::send_module_route_goodbye(
2475                    &self.counters,
2476                    &released.sink,
2477                    frame,
2478                    released.module_id.as_deref(),
2479                    "client route released",
2480                );
2481                continue;
2482            }
2483            if let Err(err) = released.sink.try_send(frame) {
2484                warn!(
2485                    target_connection_id = released.connection_id.get(),
2486                    route_channel = released.channel,
2487                    error = %err,
2488                    "route GOODBYE was not delivered to client; closing target connection"
2489                );
2490                if self
2491                    .forwarding
2492                    .escalate_client_delivery_failure(
2493                        released.connection_id,
2494                        released.channel,
2495                        released.epoch,
2496                        CloseReason::new(
2497                            "route_goodbye_delivery_failed",
2498                            format!(
2499                                "failed to enqueue route GOODBYE for channel {}: {err}",
2500                                released.channel
2501                            ),
2502                        ),
2503                        crate::forwarding::UndeliveredFrame {
2504                            module_id: released.module_id.as_deref(),
2505                            sink: &released.sink,
2506                        },
2507                    )
2508                    .unwrap_or(false)
2509                {
2510                    self.counters.increment_goodbye_relay_client_failed();
2511                }
2512            }
2513        }
2514    }
2515
2516    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2517    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2518    /// own commit failed after the module had already accepted). Without this, a
2519    /// module that accepts late keeps a binding subc has torn down, so a later
2520    /// frame on that module channel could misdeliver if the channel is reused.
2521    ///
2522    /// Never closes the shared module connection on failure: a dropped notification
2523    /// only wastes a bounded amount of warm module-side state, which the module's
2524    /// own idle reaper reclaims. Only call this once the route.bind relay was
2525    /// actually enqueued to the module — if the relay send itself failed, the
2526    /// module never created a binding and there is nothing to tear down.
2527    fn send_abandoned_route_bind_goodbye(
2528        &self,
2529        module_sink: &crate::FrameSink,
2530        negotiated_ver: u8,
2531        module_channel: u16,
2532        module_epoch: u32,
2533    ) {
2534        let frame = match Frame::build_with_version(
2535            negotiated_ver,
2536            FrameType::Goodbye,
2537            control_flags(),
2538            module_channel,
2539            module_epoch,
2540            0,
2541            Vec::new(),
2542        ) {
2543            Ok(frame) => frame,
2544            Err(err) => {
2545                warn!(
2546                    route_channel = module_channel,
2547                    error = %err,
2548                    "failed to build GOODBYE for abandoned route.bind"
2549                );
2550                return;
2551            }
2552        };
2553        crate::forwarding::send_module_route_goodbye(
2554            &self.counters,
2555            module_sink,
2556            frame,
2557            None,
2558            "abandoned route.bind",
2559        );
2560    }
2561
2562    fn handle_hello(
2563        &self,
2564        connection_id: ConnectionId,
2565        sink: Option<crate::FrameSink>,
2566        frame: Frame,
2567    ) -> Result<Vec<Frame>, RouterError> {
2568        debug!(
2569            connection_id = connection_id.get(),
2570            corr = frame.header.corr,
2571            "handling HELLO"
2572        );
2573        // A module connection has one identity for its entire lifetime. A second
2574        // registration would leave the old registry owner behind while replacing
2575        // its forwarding endpoint and launch nonce.
2576        if self
2577            .registry
2578            .get_module_by_connection(connection_id)
2579            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2580            .is_some()
2581        {
2582            return Ok(vec![control_error_frame(
2583                &frame,
2584                "invalid_hello",
2585                "connection is already registered as a module",
2586            )?]);
2587        }
2588        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2589            Ok(value) => value,
2590            Err(err) => {
2591                return Ok(vec![control_error_frame(
2592                    &frame,
2593                    "invalid_hello",
2594                    format!("malformed HELLO body: {err}"),
2595                )?])
2596            }
2597        };
2598        if let Err(err) = validate_hello_display_name(&hello_value) {
2599            return Ok(vec![control_error_frame(
2600                &frame,
2601                "invalid_manifest",
2602                err.to_string(),
2603            )?]);
2604        }
2605        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2606            return Ok(vec![control_error_frame(
2607                &frame,
2608                "invalid_capability_grammar",
2609                err.to_string(),
2610            )?]);
2611        }
2612        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2613            return Ok(vec![control_error_frame(
2614                &frame,
2615                "invalid_manifest",
2616                err.to_string(),
2617            )?]);
2618        }
2619        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2620            return Ok(vec![control_error_frame(
2621                &frame,
2622                "invalid_event_declaration",
2623                err.to_string(),
2624            )?]);
2625        }
2626        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2627            return Ok(vec![control_error_frame(
2628                &frame,
2629                "invalid_manifest",
2630                err.to_string(),
2631            )?]);
2632        }
2633        if let Some(provenance) = hello_value
2634            .get("manifest")
2635            .and_then(|manifest| manifest.get("provenance"))
2636        {
2637            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2638                return Ok(vec![control_error_frame(
2639                    &frame,
2640                    "invalid_manifest",
2641                    format!("malformed manifest provenance: {err}"),
2642                )?]);
2643            }
2644        }
2645        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2646            let module_id = hello_value["manifest"]["module_id"]
2647                .as_str()
2648                .unwrap_or("<unknown>");
2649            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2650        }
2651        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2652            let module_id = hello_value["manifest"]["module_id"]
2653                .as_str()
2654                .unwrap_or("<unknown>");
2655            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2656        }
2657        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2658            let module_id = hello_value["manifest"]["module_id"]
2659                .as_str()
2660                .unwrap_or("<unknown>");
2661            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2662        }
2663        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2664            Ok(hello) => hello,
2665            Err(err) => {
2666                return Ok(vec![control_error_frame(
2667                    &frame,
2668                    "invalid_hello",
2669                    format!("malformed HELLO body: {err}"),
2670                )?])
2671            }
2672        };
2673
2674        if hello.protocol_ver != hello.manifest.protocol_ver {
2675            return Ok(vec![control_error_frame(
2676                &frame,
2677                "invalid_manifest",
2678                format!(
2679                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2680                    hello.protocol_ver, hello.manifest.protocol_ver
2681                ),
2682            )?]);
2683        }
2684
2685        if hello.manifest.module_id.trim().is_empty() {
2686            return Ok(vec![control_error_frame(
2687                &frame,
2688                "invalid_manifest",
2689                "manifest module_id must not be empty",
2690            )?]);
2691        }
2692
2693        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2694            Ok(negotiated_ver) => negotiated_ver,
2695            Err(message) => {
2696                return Ok(vec![control_error_frame(
2697                    &frame,
2698                    "version_unsupported",
2699                    message,
2700                )?])
2701            }
2702        };
2703
2704        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2705        // swap is open for this id, the only HELLO admitted as a second process
2706        // is the one carrying the candidate's launch nonce (the swap token), and
2707        // it registers into the candidate slot rather than being refused as a
2708        // duplicate. Run after the reserved gate, a reserved module's candidate
2709        // would be refused `reserved_module` for presenting a nonce that gate
2710        // does not know. See `SupervisorHandle::swap_hello_admission`.
2711        let swap_admission = self
2712            .supervisor
2713            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2714        if swap_admission == SwapHelloAdmission::Refused {
2715            warn!(
2716                module_id = %hello.manifest.module_id,
2717                connection_id = connection_id.get(),
2718                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2719            );
2720            return Ok(vec![control_error_frame(
2721                &frame,
2722                "swap_token_invalid",
2723                format!(
2724                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2725                    hello.manifest.module_id
2726                ),
2727            )?]);
2728        }
2729        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2730
2731        // A swap candidate proves its identity with the candidate nonce above.
2732        // Other registrations receive this catalog fact only when reserved-module
2733        // policy required and accepted a nonce; unreserved registrations require
2734        // no nonce check.
2735        let nonce_verified = if swap_candidate {
2736            true
2737        } else {
2738            match self.supervisor.reserved_hello_verification(
2739                &hello.manifest.module_id,
2740                hello.launch_nonce.as_deref(),
2741            ) {
2742                Ok(verified) => verified,
2743                Err(rejection) => {
2744                    let message = match rejection {
2745                        ReservedHelloRejection::Exact { module_id } => format!(
2746                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2747                        ),
2748                        ReservedHelloRejection::Prefix {
2749                            prefix,
2750                            owner_module_id,
2751                        } => format!(
2752                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2753                            hello.manifest.module_id
2754                        ),
2755                    };
2756                    return Ok(vec![control_error_frame(
2757                        &frame,
2758                        "reserved_module",
2759                        message,
2760                    )?]);
2761                }
2762            }
2763        };
2764
2765        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2766            &hello.manifest.module_id,
2767            hello.manifest.capabilities.as_ref(),
2768        );
2769        if let Some(refusal) = reserved_capability_refusals.first() {
2770            let capability = refusal.capability.clone();
2771            let bound_module = refusal.claimants[0].clone();
2772            log_duplicate_claim_events(reserved_capability_refusals);
2773            return Ok(vec![control_error_frame(
2774                &frame,
2775                "reserved_capability",
2776                format!(
2777                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2778                    capability, bound_module, hello.manifest.module_id
2779                ),
2780            )?]);
2781        }
2782
2783        // A connection that already opened client routes must not also register as
2784        // a module: cleanup would then release only one side and leak the other.
2785        if self
2786            .forwarding
2787            .connection_has_client_routes(connection_id)
2788            .map_err(RouterError::Forwarding)?
2789        {
2790            return Ok(vec![control_error_frame(
2791                &frame,
2792                "invalid_hello",
2793                "connection has open client routes and cannot also register as a module",
2794            )?]);
2795        }
2796
2797        // Kept for scope sync authority, which goes only to the connection that
2798        // presented the module's current launch nonce. Recorded before the
2799        // registration is attempted: a connection whose registration then fails
2800        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2801        self.hello_launch_nonces
2802            .lock()
2803            .unwrap_or_else(|poisoned| poisoned.into_inner())
2804            .record(connection_id, hello.launch_nonce.as_deref());
2805        let control_ops = effective_module_control_ops(hello.control_ops);
2806        // Built before anything is registered so an encoding failure leaves no
2807        // registry or forwarding state behind.
2808        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2809        if swap_candidate {
2810            return self.register_swap_candidate(
2811                connection_id,
2812                sink,
2813                &frame,
2814                hello.manifest,
2815                negotiated_ver,
2816                control_ops,
2817                hello_ack,
2818            );
2819        }
2820        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2821        // is refused, but `catalog.list` does not show the module and the
2822        // catalog generation does not move until it is published below, once
2823        // its forwarding endpoint exists. Listing it now would advertise a
2824        // module that `route.open` still refuses for having no forwarding
2825        // connection.
2826        let registration = match self
2827            .registry
2828            .register_unpublished_verified_with_control_ops(
2829                hello.manifest,
2830                negotiated_ver,
2831                connection_id,
2832                control_ops,
2833                nonce_verified,
2834            ) {
2835            Ok(registration) => registration,
2836            Err(RegistryError::DuplicateModuleId { module_id }) => {
2837                return Ok(vec![control_error_frame(
2838                    &frame,
2839                    "duplicate_module_id",
2840                    format!(
2841                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2842                    ),
2843                )?])
2844            }
2845            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2846                return Ok(vec![control_error_frame(
2847                    &frame,
2848                    "invalid_module_id",
2849                    err.to_string(),
2850                )?])
2851            }
2852            Err(err) => {
2853                return Ok(vec![control_error_frame(
2854                    &frame,
2855                    "registry_error",
2856                    err.to_string(),
2857                )?])
2858            }
2859        };
2860        #[cfg(test)]
2861        self.hold_between_registry_and_forwarding();
2862
2863        let reply = if let Some(sink) = sink {
2864            // The forwarding table's module store is also the daemon-to-module
2865            // control-RPC lane, so every HELLO gets a live endpoint even when the
2866            // manifest has no routable provider role. Non-routable modules still
2867            // cannot receive route.bind in production: `handle_route_open` checks
2868            // the registry manifest with `target_has_required_role` before the
2869            // only production call to `begin_route_bind_relay_for` below that
2870            // route.open path. The remaining direct relay callers are unit tests
2871            // and benchmark harnesses that construct forwarding state explicitly.
2872            //
2873            // The HELLO_ACK is queued by the forwarding table itself, before the
2874            // endpoint becomes visible, and is NOT returned as a reply. A module
2875            // reads HELLO_ACK first and exits on anything else; a reply is only
2876            // written after this handler returns, by which time a route.open on
2877            // another connection could already have queued a route.bind request
2878            // for this module ahead of it.
2879            let concurrency = manifest_concurrency(&registration.manifest);
2880            if let Err(err) = self.forwarding.register_module_connection_acked(
2881                connection_id,
2882                registration.manifest.module_id.clone(),
2883                negotiated_ver,
2884                concurrency,
2885                sink,
2886                hello_ack,
2887            ) {
2888                // Forwarding registration failed, so there is no forwarding
2889                // state to tear down. Remove the registry entry, which was never
2890                // published and so never listed or counted in the catalog
2891                // generation, and signal the release watch directly.
2892                if matches!(
2893                    self.deregister_connection(
2894                        connection_id,
2895                        RegistrationEndReason::RegistrationFailed,
2896                    ),
2897                    Ok(r) if !r.is_empty()
2898                ) {
2899                    crate::supervise::notify_registration_release();
2900                }
2901                return Ok(vec![control_error_frame(
2902                    &frame,
2903                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2904                        "invalid_hello"
2905                    } else {
2906                        forwarding_error_code(&err)
2907                    },
2908                    err.to_string(),
2909                )?]);
2910            }
2911            Vec::new()
2912        } else {
2913            // No sink means no forwarding endpoint, so nothing can be routed
2914            // ahead of the ack; it goes out as the reply.
2915            vec![hello_ack]
2916        };
2917
2918        // The module is routable now (or, without a sink, as routable as it
2919        // will ever be), so the catalog may list it. Publishing bumps the
2920        // catalog generation in the same registry critical section that makes
2921        // the module listed.
2922        if !self
2923            .registry
2924            .publish_registration(connection_id)
2925            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2926        {
2927            debug!(
2928                module_id = %registration.manifest.module_id,
2929                connection_id = connection_id.get(),
2930                "registration was removed before it could be published"
2931            );
2932        }
2933
2934        // Exposure over assumption: Concurrency's serde default is pinned to the
2935        // pre-field behavior (ModuleManaged), so a management surface that is
2936        // genuinely Serial and just never declared it inherits concurrent
2937        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2938        // turns "no module has been bitten yet" into the checkable claim "no
2939        // module is exposed" -- one read of the boot log instead of a fleet
2940        // audit. Detected from the raw HELLO bytes because the serde default
2941        // deliberately erases the absent/declared distinction from the type.
2942        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2943            info!(
2944                module_id = %registration.manifest.module_id,
2945                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2946            );
2947        }
2948
2949        self.apply_registration_capabilities(&registration);
2950
2951        self.supervisor
2952            .record_activity_registration(&registration.manifest.module_id);
2953
2954        info!(
2955            module_id = %registration.manifest.module_id,
2956            module_version = %registration.manifest.module_version,
2957            negotiated_ver,
2958            routable_provider = manifest_provides_routable_role(&registration.manifest),
2959            connection_id = connection_id.get(),
2960            "module registered"
2961        );
2962
2963        Ok(reply)
2964    }
2965
2966    /// Register a HELLO the swap gate admitted into the candidate slot of the
2967    /// registry and of forwarding, where it is reachable over its own
2968    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2969    /// nothing routes to it until the supervisor cuts over.
2970    ///
2971    /// Registry first (unpublished), then forwarding, then publication, the
2972    /// same order as an ordinary HELLO; a forwarding failure removes the
2973    /// registry entry again. The swap supervisor treats a published, ready
2974    /// candidate as registered and promotes it, and promotion needs the
2975    /// candidate's forwarding endpoint, so the candidate is published only
2976    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2977    /// recompute capability verdicts or run the capability deny census
2978    /// (`enforce_capability_denies`): both are about the module in the active
2979    /// slot, which `route.open` can target. A candidate sits in the candidate
2980    /// slot, so it neither provides capabilities to other modules nor can be
2981    /// routed to until promotion moves it into the active slot.
2982    #[allow(clippy::too_many_arguments)]
2983    fn register_swap_candidate(
2984        &self,
2985        connection_id: ConnectionId,
2986        sink: Option<crate::FrameSink>,
2987        frame: &Frame,
2988        manifest: ModuleManifest,
2989        negotiated_ver: u8,
2990        control_ops: Vec<String>,
2991        hello_ack: Frame,
2992    ) -> Result<Vec<Frame>, RouterError> {
2993        let module_id = manifest.module_id.clone();
2994        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2995            manifest,
2996            negotiated_ver,
2997            connection_id,
2998            control_ops,
2999        ) {
3000            Ok(registration) => registration,
3001            Err(RegistryError::DuplicateModuleId { module_id }) => {
3002                return Ok(vec![control_error_frame(
3003                    frame,
3004                    "duplicate_module_id",
3005                    format!(
3006                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3007                    ),
3008                )?])
3009            }
3010            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3011                return Ok(vec![control_error_frame(
3012                    frame,
3013                    "invalid_module_id",
3014                    err.to_string(),
3015                )?])
3016            }
3017            Err(err) => {
3018                return Ok(vec![control_error_frame(
3019                    frame,
3020                    "registry_error",
3021                    err.to_string(),
3022                )?])
3023            }
3024        };
3025        #[cfg(test)]
3026        self.hold_between_registry_and_forwarding();
3027        let reply = if let Some(sink) = sink {
3028            // Same ordering as an ordinary HELLO: the forwarding table queues
3029            // the HELLO_ACK before the candidate endpoint is inserted, because
3030            // a module exits if its first frame after HELLO is anything else.
3031            let concurrency = manifest_concurrency(&registration.manifest);
3032            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3033                connection_id,
3034                module_id.clone(),
3035                negotiated_ver,
3036                concurrency,
3037                sink,
3038                hello_ack,
3039            ) {
3040                if matches!(
3041                    self.deregister_connection(
3042                        connection_id,
3043                        RegistrationEndReason::RegistrationFailed,
3044                    ),
3045                    Ok(r) if !r.is_empty()
3046                ) {
3047                    crate::supervise::notify_registration_release();
3048                }
3049                return Ok(vec![control_error_frame(
3050                    frame,
3051                    forwarding_error_code(&err),
3052                    err.to_string(),
3053                )?]);
3054            }
3055            Vec::new()
3056        } else {
3057            vec![hello_ack]
3058        };
3059        if !self
3060            .registry
3061            .publish_registration(connection_id)
3062            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3063        {
3064            debug!(
3065                module_id = %module_id,
3066                connection_id = connection_id.get(),
3067                "swap candidate registration was removed before it could be published"
3068            );
3069        }
3070        self.supervisor.mark_swap_candidate_admitted(&module_id);
3071        info!(
3072            module_id = %module_id,
3073            module_version = %registration.manifest.module_version,
3074            negotiated_ver,
3075            ready = registration.ready,
3076            connection_id = connection_id.get(),
3077            "swap candidate registered; not routable until cutover"
3078        );
3079        Ok(reply)
3080    }
3081
3082    fn build_hello_ack(
3083        &self,
3084        frame: &Frame,
3085        negotiated_ver: u8,
3086        module_id: &str,
3087    ) -> Result<Frame, RouterError> {
3088        let ack = ModuleHelloAckBody {
3089            negotiated_ver,
3090            subc_ops: module_subc_ops(),
3091            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3092            storage: self
3093                .storage_config
3094                .as_ref()
3095                .map(|cfg| cfg.descriptor_for(module_id)),
3096            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3097        };
3098        let body = serde_json::to_vec(&ack).map_err(|err| {
3099            RouterError::backend(
3100                0,
3101                frame.header.corr,
3102                format!("failed to encode HELLO_ACK: {err}"),
3103            )
3104        })?;
3105
3106        Frame::build_with_version(
3107            negotiated_ver,
3108            FrameType::HelloAck,
3109            control_flags(),
3110            0,
3111            0,
3112            frame.header.corr,
3113            body,
3114        )
3115        .map_err(RouterError::FrameBuild)
3116    }
3117
3118    async fn handle_client_control_request(
3119        &self,
3120        ctx: &RouteCtx,
3121        frame: Frame,
3122        request: ClientControlRequest,
3123        route_open_timings: &mut Option<RouteOpenTimings>,
3124    ) -> Result<Vec<Frame>, RouterError> {
3125        if let Some(operation) = privileged_operation(&request) {
3126            let caller = self.resolve_caller(ctx.connection_id).await;
3127            let op = client_control_request_op(&request);
3128            let text = prompt_text(operation, &caller);
3129            match self.privileged_operator_prompt {
3130                PrivilegedOperatorPrompt::Log => {
3131                    info!(target: "subc_daemon::privileged_prompt", operation = op,
3132                        target = ?operation, caller = %caller.description(), summary = %text,
3133                        "privileged request would prompt the operator");
3134                }
3135                PrivilegedOperatorPrompt::Enforce => {
3136                    let key =
3137                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3138                    let stamp = self
3139                        .operator_apps
3140                        .lock()
3141                        .unwrap_or_else(|p| p.into_inner())
3142                        .get(&ctx.connection_id)
3143                        .cloned();
3144                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3145                        &self.operator_app_policy,
3146                        &self.launch_table,
3147                        stamp.as_ref(),
3148                    )
3149                    .then(|| self.shortcut_context(stamp, None, None));
3150                    let outcome = self
3151                        .forwarding
3152                        .operator_confirms()
3153                        .approve_privileged_with_shortcut(
3154                            ctx,
3155                            &frame,
3156                            op,
3157                            text,
3158                            caller.description(),
3159                            key,
3160                            shortcut,
3161                        )
3162                        .await;
3163                    if outcome.code != "confirmed" {
3164                        return Ok(vec![outcome.frame(&frame)]);
3165                    }
3166                }
3167            }
3168        }
3169        match request {
3170            ClientControlRequest::DaemonShutdownIntent { intent } => {
3171                if intent.as_str() != Some("restart") {
3172                    return Ok(vec![control_error_frame(
3173                        &frame,
3174                        "invalid_params",
3175                        "intent must be 'restart'",
3176                    )?]);
3177                }
3178                let response = ClientControlResponse::DaemonShutdownIntent {
3179                    intent: "restart".to_owned(),
3180                    expires_at_ms: self.supervisor.announce_daemon_restart(),
3181                };
3182                Ok(vec![control_response_body_frame(
3183                    &frame,
3184                    &response,
3185                    "ClientControlResponse::DaemonShutdownIntent",
3186                )?])
3187            }
3188            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3189            ClientControlRequest::CatalogList { module_id } => {
3190                self.handle_catalog_list(frame, module_id)
3191            }
3192            ClientControlRequest::RouteOpen {
3193                target,
3194                identity,
3195                consumer_identity,
3196                consumer_capabilities,
3197                role_versions,
3198                busy_refusals,
3199                admission_facts,
3200                scope,
3201                ..
3202            } => {
3203                self.handle_route_open(
3204                    ctx,
3205                    frame,
3206                    RouteOpenRequest {
3207                        target,
3208                        identity,
3209                        consumer_identity,
3210                        consumer_capabilities,
3211                        role_versions,
3212                        busy_refusals,
3213                        admission_facts,
3214                        scope,
3215                    },
3216                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3217                )
3218                .await
3219            }
3220            ClientControlRequest::RoutePoll {
3221                route_channel,
3222                route_epoch,
3223                kind,
3224            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3225            ClientControlRequest::SupervisorList {} => {
3226                let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3227                    self.agent_declarations.refuse(
3228                        self,
3229                        ctx,
3230                        control_error_frame(
3231                            &frame,
3232                            "busy",
3233                            "supervisor list work is at capacity; retry",
3234                        )?,
3235                    );
3236                    return Ok(Vec::new());
3237                };
3238                // Filesystem-backed diagnostics must not hold up this connection's reader.
3239                let handler = self.clone();
3240                let sink = ctx.egress.clone();
3241                tokio::spawn(
3242                    async move {
3243                        let _permit = permit;
3244                        let result = handler.handle_supervisor_list(frame.clone()).await;
3245                        let replies = match result {
3246                            Ok(replies) => replies,
3247                            Err(error) => {
3248                                control_error_frame(&frame, "backend_error", error.to_string())
3249                                    .into_iter()
3250                                    .collect()
3251                            }
3252                        };
3253                        for reply in replies {
3254                            if !matches!(
3255                                tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3256                                    .await,
3257                                Ok(Ok(()))
3258                            ) {
3259                                break;
3260                            }
3261                        }
3262                    }
3263                    .with_current_subscriber(),
3264                );
3265                Ok(Vec::new())
3266            }
3267            ClientControlRequest::SupervisorSpawnSnapshot {} => {
3268                self.handle_supervisor_spawn_snapshot(frame)
3269            }
3270            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3271                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3272            }
3273            ClientControlRequest::SupervisorRestart {
3274                module_id,
3275                drain_timeout_ms,
3276            } => {
3277                self.log_supervisor_request_received(
3278                    ctx,
3279                    frame.header.corr,
3280                    ops::SUPERVISOR_RESTART,
3281                    Some(&module_id),
3282                    None,
3283                )?;
3284                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3285                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3286                    .await
3287            }
3288            ClientControlRequest::SupervisorSwap {
3289                module_id,
3290                ready_timeout_ms,
3291            } => {
3292                self.log_supervisor_request_received(
3293                    ctx,
3294                    frame.header.corr,
3295                    ops::SUPERVISOR_SWAP,
3296                    Some(&module_id),
3297                    None,
3298                )?;
3299                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3300                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3301                    .await
3302            }
3303            ClientControlRequest::SupervisorReload { module_id } => {
3304                self.log_supervisor_request_received(
3305                    ctx,
3306                    frame.header.corr,
3307                    ops::SUPERVISOR_RELOAD,
3308                    Some(&module_id),
3309                    None,
3310                )?;
3311                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3312                self.handle_supervisor_reload(frame, module_id, requester)
3313                    .await
3314            }
3315            ClientControlRequest::SupervisorRescan { preview } => {
3316                if !preview {
3317                    self.log_supervisor_request_received(
3318                        ctx,
3319                        frame.header.corr,
3320                        ops::SUPERVISOR_RESCAN,
3321                        None,
3322                        None,
3323                    )?;
3324                }
3325                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3326                let operation = self.handle_supervisor_rescan(
3327                    frame,
3328                    if preview {
3329                        RescanMode::Preview
3330                    } else {
3331                        RescanMode::Execute
3332                    },
3333                    requester,
3334                );
3335                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3336                    .await
3337            }
3338            ClientControlRequest::SupervisorRescanChecked {
3339                preview,
3340                candidate,
3341                expect,
3342            } => {
3343                self.observe_control_wait(
3344                    ctx,
3345                    None,
3346                    ops::SUPERVISOR_RESCAN,
3347                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3348                )
3349                .await
3350            }
3351            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3352                self.log_supervisor_request_received(
3353                    ctx,
3354                    frame.header.corr,
3355                    ops::SUPERVISOR_RELEASE_RESERVED,
3356                    Some(&module_id),
3357                    None,
3358                )?;
3359                self.handle_supervisor_release_reserved(frame, module_id)
3360                    .await
3361            }
3362            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3363                self.log_supervisor_request_received(
3364                    ctx,
3365                    frame.header.corr,
3366                    ops::SUPERVISOR_SET_ENABLED,
3367                    Some(&module_id),
3368                    Some(enabled),
3369                )?;
3370                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3371                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3372                    .await
3373            }
3374            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3375                self.handle_supervisor_health_probe(ctx, frame, module_id)
3376                    .await
3377            }
3378            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3379            ClientControlRequest::SupervisorRoutes { module_id } => {
3380                self.handle_supervisor_routes(frame, module_id)
3381            }
3382            ClientControlRequest::SupervisorProvenance { module_id } => {
3383                self.handle_supervisor_provenance(ctx, frame, module_id)
3384                    .await
3385            }
3386            ClientControlRequest::SupervisorStderrTail {
3387                module_id,
3388                max_lines,
3389                max_bytes,
3390            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3391            ClientControlRequest::SupervisorTerminals { module_id } => {
3392                self.handle_supervisor_terminals(ctx, frame, module_id)
3393                    .await
3394            }
3395        }
3396    }
3397
3398    fn log_supervisor_request_received(
3399        &self,
3400        ctx: &RouteCtx,
3401        corr: u64,
3402        op: &'static str,
3403        module_id: Option<&str>,
3404        enabled: Option<bool>,
3405    ) -> Result<(), RouterError> {
3406        let caller = self.supervisor_requester(ctx, corr)?;
3407        let caller = principal_label(&caller);
3408        // `Option` fields are recorded only when present, so a request with no
3409        // module id (rescan) or no enabled flag simply omits that field.
3410        info!(
3411            target: "control",
3412            op,
3413            module_id,
3414            enabled,
3415            connection_id = ctx.connection_id.get(),
3416            caller = %caller,
3417            "supervisor request received"
3418        );
3419        Ok(())
3420    }
3421
3422    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3423        Ok(self
3424            .registry
3425            .get_module_by_connection(ctx.connection_id)
3426            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3427            .map(|registration| Principal::Reserved {
3428                module_id: registration.manifest.module_id,
3429            })
3430            .unwrap_or(Principal::Direct))
3431    }
3432
3433    fn handle_module_control_request(
3434        &self,
3435        connection_id: ConnectionId,
3436        frame: Frame,
3437        request: ModuleControlRequestFromModule,
3438    ) -> Result<Vec<Frame>, RouterError> {
3439        match request {
3440            ModuleControlRequestFromModule::CatalogUpdate {
3441                provides,
3442                capabilities,
3443                ready,
3444            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3445            ModuleControlRequestFromModule::LiveRoots {} => {
3446                let registered = self
3447                    .registry
3448                    .get_module_by_connection(connection_id)
3449                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3450                let Some(registration) = registered else {
3451                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3452                };
3453                let response = self
3454                    .forwarding
3455                    .live_roots(&registration.manifest.module_id)
3456                    .map_err(RouterError::Forwarding)?;
3457                Ok(vec![control_response_body_frame(
3458                    &frame,
3459                    &response,
3460                    "ModuleControlResponseToModule::LiveRoots",
3461                )?])
3462            }
3463            ModuleControlRequestFromModule::AgentCommands {
3464                started,
3465                finished,
3466                known,
3467            } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3468            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3469                self.handle_scope_sync(connection_id, frame, generation, scopes)
3470            }
3471            ModuleControlRequestFromModule::ScopeApply {
3472                generation,
3473                upsert,
3474                end,
3475            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3476            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3477                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3478            }
3479            _ => Ok(vec![control_error_frame(
3480                &frame,
3481                "unsupported_control_frame",
3482                "unsupported module control operation",
3483            )?]),
3484        }
3485    }
3486
3487    fn handle_scope_served_by(
3488        &self,
3489        connection_id: ConnectionId,
3490        frame: Frame,
3491    ) -> Result<Vec<Frame>, RouterError> {
3492        let registered = self
3493            .registry
3494            .get_module_by_connection(connection_id)
3495            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3496        let Some(registered) = registered else {
3497            return Ok(vec![control_error_frame(
3498                &frame,
3499                "not_registered",
3500                "scope.served_by requires an active module registration owned by this connection",
3501            )?]);
3502        };
3503        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3504        {
3505            Ok(request) => request,
3506            Err(err) => {
3507                return Ok(vec![control_error_frame(
3508                    &frame,
3509                    "invalid_control_body",
3510                    format!("malformed scope.served_by body: {err}"),
3511                )?])
3512            }
3513        };
3514        let answer = self
3515            .scopes
3516            .read()
3517            .unwrap_or_else(|poisoned| poisoned.into_inner())
3518            .served_by(
3519                &registered.manifest.module_id,
3520                &request.scope_ref,
3521                request.scope_epoch,
3522                &request.module_id,
3523            );
3524        let reply = subc_control::ScopeServedByReply::new(
3525            answer,
3526            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3527        );
3528        Ok(vec![control_response_body_frame(
3529            &frame,
3530            &reply,
3531            "ScopeServedByReply",
3532        )?])
3533    }
3534
3535    fn handle_operator_confirm(
3536        &self,
3537        ctx: &RouteCtx,
3538        frame: Frame,
3539    ) -> Result<Vec<Frame>, RouterError> {
3540        use crate::operator_confirm::{audit, Outcome};
3541        let registration = self
3542            .registry
3543            .get_module_by_connection(ctx.connection_id)
3544            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3545        let Some(registration) = registration else {
3546            let outcome = Outcome::refusal("not_registered");
3547            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3548            return Ok(vec![outcome.frame(&frame)]);
3549        };
3550        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3551            Ok(request) => request,
3552            Err(_) => {
3553                let outcome = Outcome::refusal("invalid_control_body");
3554                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3555                return Ok(vec![outcome.frame(&frame)]);
3556            }
3557        };
3558        let declared = matches!(
3559            request.kind,
3560            Some(subc_protocol::session::OperatorConfirmKind::Operation)
3561        ) && request.operation.as_ref().is_some_and(|operation| {
3562            registration
3563                .manifest
3564                .confirm_operations
3565                .as_ref()
3566                .is_some_and(|operations| operations.contains(operation))
3567        });
3568        let module_id = registration.manifest.module_id;
3569        // Read the launch nonce (under its own lock) before taking the forwarding
3570        // table's lock below: holding forwarding while waiting on another daemon
3571        // lock risks a lock-order deadlock with paths that take them the other way.
3572        let nonce = self
3573            .hello_launch_nonces
3574            .lock()
3575            .unwrap_or_else(|p| p.into_inner())
3576            .nonce(ctx.connection_id)
3577            .map(str::to_owned);
3578        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3579            self.supervisor
3580                .spawned_consumer_authorized(&module_id, nonce)
3581        });
3582        let confirms = self.forwarding.operator_confirms();
3583        self.forwarding
3584            .with_operator_route(
3585                ctx.connection_id,
3586                request.route_channel,
3587                request.route_epoch,
3588                |binding| {
3589                    let shortcut = binding
3590                        .filter(|binding| {
3591                            nonce_proven
3592                                && crate::operator_confirm::valid_summary(&request.summary)
3593                                && declared
3594                                && crate::operator_confirm::ShortcutContext::route_eligible(
3595                                    &self.operator_app_policy,
3596                                    &self.launch_table,
3597                                    binding,
3598                                )
3599                        })
3600                        .map(|binding| {
3601                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3602                                &self.operator_app_policy,
3603                                binding,
3604                            );
3605                            let stamp = if phone.is_some() {
3606                                None
3607                            } else {
3608                                binding.operator_app.clone()
3609                            };
3610                            self.shortcut_context(
3611                                stamp,
3612                                phone,
3613                                Some((request.route_channel, request.route_epoch)),
3614                            )
3615                        });
3616                    confirms.admit_with_shortcut(
3617                        ctx,
3618                        frame,
3619                        module_id,
3620                        nonce_proven,
3621                        request,
3622                        binding,
3623                        shortcut,
3624                    )
3625                },
3626            )
3627            .map_err(RouterError::Forwarding)
3628    }
3629
3630    fn shortcut_context(
3631        &self,
3632        stamp: Option<crate::operator_apps::OperatorAppStamp>,
3633        phone_fingerprint: Option<String>,
3634        route: Option<(u16, u32)>,
3635    ) -> crate::operator_confirm::ShortcutContext {
3636        crate::operator_confirm::ShortcutContext {
3637            ledger: self.activity.clone(),
3638            forwarding: self.forwarding.clone(),
3639            launches: self.launch_table.clone(),
3640            stamp,
3641            phone_fingerprint,
3642            route,
3643            connections: self.operator_apps.clone(),
3644            #[cfg(test)]
3645            hooks: Default::default(),
3646        }
3647    }
3648
3649    /// Explicit test-only phone policy; daemon configuration cannot select it.
3650    #[cfg(any(test, feature = "test-support"))]
3651    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3652        self.operator_app_policy.phone_enabled = enabled;
3653        self
3654    }
3655
3656    /// Resolve folders before taking the scope-table lock: a slow filesystem
3657    /// must not hold up admissions for unrelated scoped routes.
3658    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3659        let mut scopes = ScopeRecords::from(scopes);
3660        for (index, record) in scopes.records.iter_mut().enumerate() {
3661            if let Some(refusal) = Self::canonicalize_workdir(record) {
3662                scopes.refusals.insert(index, refusal);
3663            }
3664        }
3665        scopes
3666    }
3667
3668    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3669        let workdir = record.workdir.as_mut()?;
3670        let path = std::path::Path::new(workdir);
3671        if !path.is_absolute() {
3672            return Some((
3673                error_codes::INVALID_WORKDIR,
3674                "workdir must be absolute".to_string(),
3675            ));
3676        }
3677        match ProjectRootId::from_path_allowing_missing(path) {
3678            Ok(root) => match root.as_path().to_str() {
3679                Some(canonical) => {
3680                    *workdir = canonical.to_string();
3681                    None
3682                }
3683                None => Some((
3684                    error_codes::INVALID_WORKDIR,
3685                    "canonical workdir is not UTF-8".to_string(),
3686                )),
3687            },
3688            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3689        }
3690    }
3691
3692    /// `scope.sync`: the owner is the module registered on this connection.
3693    /// A connection with no registration (every client connection, `direct`
3694    /// included) is refused `not_registered` before the table is consulted.
3695    fn handle_scope_sync(
3696        &self,
3697        connection_id: ConnectionId,
3698        frame: Frame,
3699        generation: u64,
3700        scopes: Vec<ScopeRecord>,
3701    ) -> Result<Vec<Frame>, RouterError> {
3702        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3703    }
3704
3705    fn handle_scope_change(
3706        &self,
3707        connection_id: ConnectionId,
3708        frame: Frame,
3709        generation: u64,
3710        scopes: Vec<ScopeRecord>,
3711        end: Option<Vec<ScopeEnd>>,
3712    ) -> Result<Vec<Frame>, RouterError> {
3713        let is_apply = end.is_some();
3714        let Some(registration) = self
3715            .registry
3716            .get_module_by_connection(connection_id)
3717            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3718        else {
3719            return Ok(vec![control_error_frame(
3720                &frame,
3721                "not_registered",
3722                "scope.sync and scope.apply require an active module registration owned by this connection",
3723            )?]);
3724        };
3725        let owner = registration.manifest.module_id;
3726        let scopes = Self::prepare_scope_records(scopes);
3727        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3728        let is_current_launch = |connection: ConnectionId| {
3729            self.hello_launch_nonces
3730                .lock()
3731                .unwrap_or_else(|poisoned| poisoned.into_inner())
3732                .presented(connection, current_nonce.as_deref())
3733        };
3734        // Lock order is the scope table, then the forwarding table: the new
3735        // tags are published, and the routes the change closes are selected,
3736        // while the scope table is still write-locked, so no admission can read
3737        // a record whose tag is not yet published.
3738        let mut table = self
3739            .scopes
3740            .write()
3741            .unwrap_or_else(|poisoned| poisoned.into_inner());
3742        let outcome = match end {
3743            Some(end) => table.apply(
3744                &owner,
3745                connection_id,
3746                is_current_launch,
3747                generation,
3748                scopes,
3749                end,
3750            ),
3751            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3752        };
3753        let drained = match &outcome {
3754            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3755            Err(_) => Vec::new(),
3756        };
3757        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3758        drop(table);
3759        if has_deadlines {
3760            self.scope_deadline_added.notify_one();
3761        }
3762        match outcome {
3763            Ok(applied) => {
3764                let counts = ScopeOutcomeCounts::of(&applied.results);
3765                info!(
3766                    owner = %owner,
3767                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3768                    generation,
3769                    records = applied.results.len(),
3770                    created = counts.created,
3771                    replaced = counts.replaced,
3772                    updated = counts.updated,
3773                    unchanged = counts.unchanged,
3774                    refused = counts.refused,
3775                    ended = applied.ended.len(),
3776                    tag_changes = applied.tag_changes.len(),
3777                    routes_closed = drained.len(),
3778                    "scope change accepted"
3779                );
3780                // An accepted scope change can still refuse individual records, and the
3781                // owner is the only party that sees the reply. Name them here so
3782                // an operator can tell a refused session from a missing one
3783                // without the owner's logs. Capped so a call that refuses
3784                // thousands cannot flood the log; the count above is complete.
3785                for refused in applied
3786                    .results
3787                    .iter()
3788                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3789                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3790                {
3791                    warn!(
3792                        owner = %owner,
3793                        generation,
3794                        scope_ref = %refused.scope_ref,
3795                        scope_epoch = refused.scope_epoch,
3796                        code = refused.code.as_deref().unwrap_or(""),
3797                        "scope record refused"
3798                    );
3799                }
3800                self.close_scope_drained_routes(drained);
3801                let response = if is_apply {
3802                    ModuleControlResponseToModule::ScopeApply {
3803                        generation,
3804                        results: applied.results,
3805                        end_results: applied.end_results,
3806                        ended: applied.ended,
3807                    }
3808                } else {
3809                    ModuleControlResponseToModule::ScopeSync {
3810                        generation,
3811                        results: applied.results,
3812                        ended: applied.ended,
3813                    }
3814                };
3815                Ok(vec![control_response_body_frame(
3816                    &frame,
3817                    &response,
3818                    "scope change response",
3819                )?])
3820            }
3821            Err(refusal) => {
3822                info!(
3823                    owner = %owner,
3824                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3825                    generation,
3826                    code = refusal.code,
3827                    "scope change refused"
3828                );
3829                Ok(vec![control_error_frame(
3830                    &frame,
3831                    refusal.code,
3832                    refusal.message,
3833                )?])
3834            }
3835        }
3836    }
3837
3838    /// Tell both ends of each route a scope change closed. The module gets a
3839    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3840    /// the client's route handle. The client also gets `route.closed` with the
3841    /// scope reason, one push per module and reason, so it can tell a revoked
3842    /// route from an ordinary close and not reopen it.
3843    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3844        if drained.is_empty() {
3845            return;
3846        }
3847        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3848            BTreeMap::new();
3849        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3850        for route in drained {
3851            warn!(
3852                module_id = %route.module_id,
3853                reason = ?route.reason,
3854                client_connection_id = route.client.connection_id.get(),
3855                route_channel = route.client.channel,
3856                "closing route because its scope changed"
3857            );
3858            pushes
3859                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3860                .or_insert_with(|| (route.reason, Vec::new()))
3861                .1
3862                .push(EndpointRoute {
3863                    goodbye_target: route.client.clone(),
3864                    principal: Principal::Unverified,
3865                    bound_at: Instant::now(),
3866                    draining: false,
3867                    drain_reason: None,
3868                });
3869            goodbyes.push(route.module);
3870            goodbyes.push(route.client);
3871        }
3872        for ((module_id, _), (reason, routes)) in pushes {
3873            send_route_control_pushes(
3874                &self.forwarding,
3875                routes,
3876                ClientControlPush::RouteClosed {
3877                    module_id,
3878                    channels: Vec::new(),
3879                    reason,
3880                    drained: false,
3881                    abandoned: 0,
3882                    excluded_subscriptions: 0,
3883                    terminal: Some(false),
3884                },
3885            );
3886        }
3887        self.emit_route_goodbyes(goodbyes);
3888    }
3889
3890    /// `scope.describe`: any registered module may read any scope, because a
3891    /// provider must read the scope a route it serves is stamped with.
3892    fn handle_scope_describe(
3893        &self,
3894        connection_id: ConnectionId,
3895        frame: Frame,
3896        owner: Principal,
3897        scope_ref: String,
3898    ) -> Result<Vec<Frame>, RouterError> {
3899        let registered = self
3900            .registry
3901            .get_module_by_connection(connection_id)
3902            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3903        let Some(registered) = registered else {
3904            return Ok(vec![control_error_frame(
3905                &frame,
3906                "not_registered",
3907                "scope.describe requires an active module registration owned by this connection",
3908            )?]);
3909        };
3910        let scopes = self
3911            .scopes
3912            .read()
3913            .unwrap_or_else(|poisoned| poisoned.into_inner());
3914        if scopes.is_federated_owner(&owner)
3915            && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
3916        {
3917            return Ok(vec![control_error_frame(
3918                &frame,
3919                error_codes::SCOPE_FEDERATED_UNSUPPORTED,
3920                "scope.describe of the federated owner requires scope-federated/v1",
3921            )?]);
3922        }
3923        let mut description = scopes.describe(&owner, &scope_ref);
3924        drop(scopes);
3925        // Describe readers also decode strict ScopeAttributes. Attribution is
3926        // safe to omit unless the requesting module declares it understands it.
3927        if !registered
3928            .manifest
3929            .capabilities
3930            .as_ref()
3931            .is_some_and(|caps| {
3932                caps.provides
3933                    .iter()
3934                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3935            })
3936        {
3937            if let Some(stamp) = description.stamp.as_mut() {
3938                stamp.attributes.on_behalf_of_agent = None;
3939            }
3940        }
3941        if !registered
3942            .manifest
3943            .capabilities
3944            .as_ref()
3945            .is_some_and(|caps| {
3946                caps.provides
3947                    .iter()
3948                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3949            })
3950        {
3951            if let Some(stamp) = description.stamp.as_mut() {
3952                stamp.attributes.flow_version = None;
3953                stamp.attributes.flow_code_hash = None;
3954                stamp.attributes.flow_install_id = None;
3955            }
3956        }
3957        let owner_configured = match &owner {
3958            // Ask whether the owner is configured (`is_configured`), not
3959            // whether it is on the roster (`get(..).is_some()`): a supervised
3960            // module's process can register and describe a scope before the
3961            // supervisor has put it on the roster.
3962            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3963            _ => false,
3964        };
3965        let response = ModuleControlResponseToModule::ScopeDescribe {
3966            status: description.status,
3967            scope_epoch: description.scope_epoch,
3968            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3969            owner_synced: description.owner_synced,
3970            owner_configured,
3971            scope: description.stamp,
3972        };
3973        Ok(vec![control_response_body_frame(
3974            &frame,
3975            &response,
3976            "ModuleControlResponseToModule::ScopeDescribe",
3977        )?])
3978    }
3979
3980    fn handle_catalog_update(
3981        &self,
3982        connection_id: ConnectionId,
3983        frame: Frame,
3984        provides: Vec<ProviderRole>,
3985        capabilities: Option<CapabilityDeclarations>,
3986        ready: Option<bool>,
3987    ) -> Result<Vec<Frame>, RouterError> {
3988        self.refresh_capability_requirements();
3989        let Some(registration) = self
3990            .registry
3991            .get_module_by_connection(connection_id)
3992            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3993        else {
3994            return Ok(vec![control_error_frame(
3995                &frame,
3996                "not_registered",
3997                "catalog.update requires an active module registration owned by this connection",
3998            )?]);
3999        };
4000
4001        if let Some(message) =
4002            catalog_update_frozen_field_message(&registration.manifest, &provides)
4003        {
4004            return Ok(vec![control_error_frame(
4005                &frame,
4006                "catalog_update_frozen_field",
4007                message,
4008            )?]);
4009        }
4010
4011        let mut candidate = registration.manifest.clone();
4012        candidate.provides = provides.clone();
4013        candidate.capabilities = capabilities
4014            .clone()
4015            .or_else(|| registration.manifest.capabilities.clone());
4016        if let Some(rich_ui) = &candidate.rich_ui {
4017            if let Err(err) = rich_ui.validate() {
4018                return Ok(vec![control_error_frame(
4019                    &frame,
4020                    "invalid_manifest",
4021                    err.to_string(),
4022                )?]);
4023            }
4024        }
4025        if let Err(err) = candidate.validate_capability_grammar() {
4026            return Ok(vec![control_error_frame(
4027                &frame,
4028                "invalid_capability_grammar",
4029                err.to_string(),
4030            )?]);
4031        }
4032
4033        // Updates must honor the same reserved owner as initial registration;
4034        // otherwise an empty HELLO could acquire the claim after admission.
4035        let mut conflicts = self
4036            .capability_evaluator
4037            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4038        if let Some(conflict) = conflicts.first() {
4039            let message = format!(
4040                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4041                conflict.capability, conflict.claimants[0], candidate.module_id
4042            );
4043            for conflict in &mut conflicts {
4044                conflict.source = DuplicateClaimSource::CatalogUpdate;
4045            }
4046            log_duplicate_claim_events(conflicts);
4047            return Ok(vec![control_error_frame(
4048                &frame,
4049                "reserved_capability",
4050                message,
4051            )?]);
4052        }
4053
4054        let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4055            Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4056                &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4057                "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4058            )?]),
4059            result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4060        };
4061        if updated.is_none() {
4062            return Ok(vec![control_error_frame(
4063                &frame,
4064                "not_registered",
4065                "catalog.update requires an active module registration owned by this connection",
4066            )?]);
4067        }
4068        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4069            log_duplicate_claim_events(
4070                self.capability_evaluator
4071                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
4072            );
4073        }
4074        if capability_census_trigger(
4075            registration.manifest.capabilities.as_ref(),
4076            updated
4077                .as_ref()
4078                .and_then(|entry| entry.manifest.capabilities.as_ref()),
4079        ) {
4080            self.enforce_capability_denies();
4081        }
4082        self.refresh_capability_requirements();
4083
4084        let response = ModuleControlResponseToModule::CatalogUpdate {};
4085        control_response_body_frame(
4086            &frame,
4087            &response,
4088            "ModuleControlResponseToModule::CatalogUpdate",
4089        )
4090        .map(|frame| vec![frame])
4091    }
4092
4093    pub(crate) fn with_activity_ledger_config(
4094        mut self,
4095        run_dir: &Path,
4096        incarnation: String,
4097        config: crate::activity::ActivityConfig,
4098    ) -> Self {
4099        self.activity = Some(crate::activity::ActivityLedger::open_config(
4100            run_dir.join("activity.jsonl"),
4101            incarnation,
4102            config,
4103        ));
4104        self.supervisor.set_activity(self.activity.clone());
4105        self
4106    }
4107
4108    pub(crate) async fn configure_activity(
4109        &self,
4110        config: crate::activity::ActivityConfig,
4111    ) -> Result<(), &'static str> {
4112        match &self.activity {
4113            Some(ledger) => ledger.configure(config).await,
4114            None => Ok(()),
4115        }
4116    }
4117
4118    fn cancel_activity_subscription(
4119        &self,
4120        ctx: &RouteCtx,
4121        frame: &Frame,
4122    ) -> Result<bool, RouterError> {
4123        let task = self
4124            .activity_streams
4125            .lock()
4126            .unwrap_or_else(|p| p.into_inner())
4127            .remove(&(ctx.connection_id, frame.header.corr));
4128        let Some(task) = task else {
4129            return Ok(false);
4130        };
4131        task.abort();
4132        let end = Frame::build_with_version(
4133            response_version(frame),
4134            FrameType::StreamEnd,
4135            control_flags(),
4136            0,
4137            0,
4138            frame.header.corr,
4139            Vec::new(),
4140        )
4141        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4142        let sink = ctx.egress.clone();
4143        tokio::spawn(async move {
4144            let _ = sink.send(end).await;
4145        });
4146        Ok(true)
4147    }
4148
4149    async fn handle_activity_read(
4150        &self,
4151        ctx: &RouteCtx,
4152        frame: Frame,
4153    ) -> Result<Vec<Frame>, RouterError> {
4154        use subc_control::{
4155            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4156            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4157        };
4158        #[derive(Deserialize)]
4159        struct Identity {
4160            #[serde(default)]
4161            consumer_identity: Option<ConsumerIdentity>,
4162        }
4163        // A registered module connection already identifies its caller. Ignore
4164        // consumer_identity there so a module cannot borrow a mirror's identity.
4165        // Only separate consumer connections use the Direct/nonce trust boundary.
4166        let principal = match self
4167            .forwarding
4168            .module_id_for_connection(ctx.connection_id)
4169            .map_err(RouterError::Forwarding)?
4170        {
4171            Some(module_id) => Ok(Principal::Reserved { module_id }),
4172            None => {
4173                let identity: Identity = match serde_json::from_slice(&frame.body) {
4174                    Ok(identity) => identity,
4175                    Err(_) => {
4176                        return Ok(vec![control_error_frame(
4177                            &frame,
4178                            ACTIVITY_MALFORMED,
4179                            "malformed activity read",
4180                        )?])
4181                    }
4182                };
4183                self.route_open_principal(&frame, identity.consumer_identity)?
4184            }
4185        };
4186        let admitted = matches!(principal, Ok(Principal::Direct))
4187            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4188        if !admitted {
4189            return Ok(vec![control_error_frame(
4190                &frame,
4191                ACTIVITY_READ_FORBIDDEN,
4192                "activity reads require an operator or trusted mirror principal",
4193            )?]);
4194        }
4195        let Some(ledger) = &self.activity else {
4196            return Ok(vec![control_error_frame(
4197                &frame,
4198                ACTIVITY_UNAVAILABLE,
4199                "activity ledger unavailable",
4200            )?]);
4201        };
4202        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4203            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4204        {
4205            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4206                Ok(request) => ledger.query(request).await,
4207                Err(_) => Err(ACTIVITY_MALFORMED),
4208            };
4209            return match result {
4210                Ok(reply) => Ok(vec![control_response_body_frame(
4211                    &frame,
4212                    &reply,
4213                    ops::ACTIVITY_QUERY,
4214                )?]),
4215                Err(code) => Ok(vec![control_error_frame(
4216                    &frame,
4217                    code,
4218                    "activity query refused",
4219                )?]),
4220            };
4221        }
4222        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4223            Ok(request) => request,
4224            Err(_) => {
4225                return Ok(vec![control_error_frame(
4226                    &frame,
4227                    ACTIVITY_MALFORMED,
4228                    "malformed activity subscription",
4229                )?])
4230            }
4231        };
4232        let mut subscription = match ledger.subscribe(request.since).await {
4233            Ok(subscription) => subscription,
4234            Err(code) => {
4235                return Ok(vec![control_error_frame(
4236                    &frame,
4237                    code,
4238                    "activity subscription refused",
4239                )?])
4240            }
4241        };
4242        let key = (ctx.connection_id, frame.header.corr);
4243        let sink = ctx.egress.clone();
4244        let mut streams = self
4245            .activity_streams
4246            .lock()
4247            .unwrap_or_else(|p| p.into_inner());
4248        streams.retain(|_, task| !task.is_finished());
4249        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4250            return Ok(vec![control_error_frame(
4251                &frame,
4252                "duplicate_subscription",
4253                "activity subscription correlation id already in use",
4254            )?]);
4255        }
4256        let task = tokio::spawn(async move {
4257            loop {
4258                let entry = if let Some(entry) = subscription.replay.pop_front() {
4259                    Some(entry)
4260                } else {
4261                    subscription.receiver.recv().await
4262                };
4263                let Some(entry) = entry else {
4264                    break;
4265                };
4266                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4267                    return;
4268                };
4269                let Ok(next) = Frame::build_with_version(
4270                    response_version(&frame),
4271                    FrameType::StreamData,
4272                    control_flags(),
4273                    0,
4274                    0,
4275                    frame.header.corr,
4276                    body,
4277                ) else {
4278                    return;
4279                };
4280                if sink.send(next).await.is_err() {
4281                    return;
4282                }
4283            }
4284            // Dropping a full subscriber's sender drains its accepted entries
4285            // before the terminal error, preserving their order on the wire.
4286            if let Some(code) = subscription.terminal_code() {
4287                if let Ok(end) =
4288                    control_error_frame(&frame, code, "activity subscriber fell behind")
4289                {
4290                    let _ = sink.send(end).await;
4291                }
4292            }
4293        });
4294        streams.insert(key, task.abort_handle());
4295        Ok(Vec::new())
4296    }
4297
4298    async fn handle_activity_record(
4299        &self,
4300        ctx: &RouteCtx,
4301        frame: Frame,
4302    ) -> Result<Vec<Frame>, RouterError> {
4303        use subc_control::{
4304            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4305            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4306        };
4307        let result = async {
4308            let module = self
4309                .forwarding
4310                .module_id_for_connection(ctx.connection_id)
4311                .ok()
4312                .flatten()
4313                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4314            let request: ActivityRecordRequest =
4315                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4316            crate::activity::validate(&module, &request)?;
4317            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4318            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4319            let at_ms = std::time::SystemTime::now()
4320                .duration_since(std::time::UNIX_EPOCH)
4321                .unwrap_or_default()
4322                .as_millis() as u64;
4323            ledger.record(module, request, actor, at_ms).await
4324        }
4325        .await;
4326        match result {
4327            Ok(entry) => Ok(vec![control_response_body_frame(
4328                &frame,
4329                &ActivityRecordResponse { seq: entry.seq },
4330                "activity.record",
4331            )?]),
4332            Err(code) => Ok(vec![control_error_frame(
4333                &frame,
4334                code,
4335                "activity recording refused",
4336            )?]),
4337        }
4338    }
4339
4340    fn activity_actor(
4341        &self,
4342        connection: ConnectionId,
4343        module: &str,
4344        on: Option<&subc_control::ActivityOn>,
4345    ) -> Result<subc_control::ActivityActor, &'static str> {
4346        use crate::forwarding::{DataRoute, DataRouteState};
4347        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4348        let principal = |id: &str| Principal::Reserved {
4349            module_id: id.to_owned(),
4350        };
4351        let mut actor = ActivityActor {
4352            principal: principal(module),
4353            scope_owner: None,
4354            agent_id: None,
4355            run_id: None,
4356            flow_id: None,
4357            project_root: None,
4358            workdir: None,
4359            project_id: None,
4360        };
4361        let scope = match on {
4362            None => return Ok(actor),
4363            Some(ActivityOn::Scope(scope)) => {
4364                let selector = subc_protocol::scope::ScopeSelector::new(
4365                    principal(module),
4366                    scope.scope_ref.clone(),
4367                    Some(scope.epoch),
4368                );
4369                self.scopes
4370                    .read()
4371                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4372                    .attribute(&principal(module), module, &selector, true)
4373                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4374            }
4375            Some(ActivityOn::Route(route)) => {
4376                let DataRoute::Module(DataRouteState::Bound(binding)) = self
4377                    .forwarding
4378                    .lookup_data_route(connection, route.channel, route.epoch)
4379                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4380                else {
4381                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4382                };
4383                if binding.module_id != module {
4384                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4385                }
4386                actor.principal = binding.principal.clone();
4387                actor.project_root = binding
4388                    .project_root
4389                    .as_ref()
4390                    .map(|root| root.as_path().to_path_buf());
4391                let Some(bound) = &binding.scope else {
4392                    return Ok(actor);
4393                };
4394                let owner = principal(&bound.owner);
4395                let selector = subc_protocol::scope::ScopeSelector::new(
4396                    owner.clone(),
4397                    bound.scope_ref.clone(),
4398                    Some(bound.tag.scope_epoch),
4399                );
4400                let admitted = self
4401                    .scopes
4402                    .read()
4403                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4404                    .attribute(&owner, module, &selector, true)
4405                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4406                if admitted.tag != bound.tag {
4407                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4408                }
4409                admitted
4410            }
4411        };
4412        actor.scope_owner = Some(scope.stamp.owner);
4413        actor.agent_id = scope.stamp.attributes.agent_id;
4414        actor.run_id = scope.stamp.attributes.run_id;
4415        actor.flow_id = scope.stamp.attributes.flow_id;
4416        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4417        actor.project_id = scope.stamp.project_id;
4418        Ok(actor)
4419    }
4420
4421    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4422        self.refresh_capability_requirements();
4423        // A bare connection count is ambiguous between many clients holding a
4424        // route each and one client accumulating hundreds, so publish the
4425        // concentration alongside it. Route state is best-effort here: a
4426        // diagnostic endpoint must still answer if the forwarding lock is
4427        // contended.
4428        let mut counters = self.counters.snapshot();
4429        let losses = self.resource_tracker.loss_diagnostics();
4430        if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4431            if let Some(obj) = counters.as_object_mut() {
4432                obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4433            }
4434        }
4435        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4436            obj.insert(
4437                "activity_unparseable_lines".into(),
4438                ledger.read_errors().into(),
4439            );
4440            obj.insert(
4441                "activity_lifecycle_append_failures".into(),
4442                ledger.lifecycle_failures().into(),
4443            );
4444            obj.insert(
4445                "activity_started_digest_skips".into(),
4446                ledger.lifecycle_digest_skips().into(),
4447            );
4448        }
4449        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4450            self.forwarding.client_route_concentration(),
4451            counters.as_object_mut(),
4452        ) {
4453            obj.insert(
4454                "client_connections_with_routes".into(),
4455                connections_with_routes.into(),
4456            );
4457            obj.insert("max_routes_on_one_connection".into(), max.into());
4458        }
4459        // A module that is being fast-refused and a module that is fine look
4460        // identical from a client that retries and succeeds, so name the open
4461        // breakers here. This rides the existing free-form counters object
4462        // rather than a new wire field, so no sibling that deserializes
4463        // `ServerDescribe` has to be rebuilt to keep reading it.
4464        if let (Some(open_breakers), Some(obj)) = (
4465            self.route_bind_breakers.open_snapshot(),
4466            counters.as_object_mut(),
4467        ) {
4468            obj.insert("route_bind_breakers_open".into(), open_breakers);
4469        }
4470        // A daemon whose local endpoint could not be bound keeps serving TCP,
4471        // and the connection file names no endpoint. Name the reason here, in
4472        // the free-form counters object, so an operator reading daemon facts
4473        // sees it without searching the log, and no wire field changes.
4474        if let (Some(reason), Some(obj)) = (
4475            self.local_endpoint_unavailable.clone(),
4476            counters.as_object_mut(),
4477        ) {
4478            obj.insert("local_endpoint_unavailable".into(), reason.into());
4479        }
4480        let response = ClientControlResponse::ServerDescribe {
4481            protocol_ver: PROTOCOL_VERSION,
4482            subc_ops: subc_ops(),
4483            capabilities: self.describe_capabilities(),
4484            connected_clients: self.connected_clients.count(),
4485            counters: Some(counters),
4486            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4487            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4488            capability_requirements: self.capability_requirement_statuses(),
4489            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4490            config_path: self
4491                .rescan
4492                .as_ref()
4493                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4494        };
4495        Ok(vec![control_response_body_frame(
4496            &frame,
4497            &response,
4498            "ClientControlResponse::ServerDescribe",
4499        )?])
4500    }
4501
4502    fn handle_catalog_list(
4503        &self,
4504        frame: Frame,
4505        module_id: Option<String>,
4506    ) -> Result<Vec<Frame>, RouterError> {
4507        let (generation, modules) = self
4508            .registry
4509            .list_modules_with_nonce_verification()
4510            .map_err(|err| {
4511                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4512            })?;
4513        let entries = modules
4514            .into_iter()
4515            .filter(|(registration, _)| {
4516                module_id
4517                    .as_deref()
4518                    .map(|wanted| registration.manifest.module_id == wanted)
4519                    .unwrap_or(true)
4520            })
4521            .map(|(registration, nonce_verified)| {
4522                let not_ready = self.not_ready_reason(&registration);
4523                let roles = registration.manifest.provides;
4524                CatalogEntry::new(
4525                    registration.manifest.module_id,
4526                    roles,
4527                    registration.control_ops,
4528                )
4529                .with_ready(not_ready.is_none())
4530                .with_display_name(registration.manifest.display_name)
4531                .with_not_ready(not_ready)
4532                .with_module_version(Some(registration.manifest.module_version))
4533                .with_provenance(registration.manifest.provenance)
4534                .with_capabilities(registration.manifest.capabilities)
4535                .with_self_signals(registration.manifest.self_signals)
4536                .with_events(registration.manifest.events)
4537                .with_operation_delivery(registration.manifest.operation_delivery)
4538                .with_operation_payload(registration.manifest.operation_payload)
4539                .with_flow_operations(registration.manifest.flow_operations)
4540                .with_confirm_operations(registration.manifest.confirm_operations)
4541                .with_rich_ui(registration.manifest.rich_ui)
4542                .with_role_declarations(registration.manifest.role_declarations)
4543                .with_nonce_verified(nonce_verified)
4544            })
4545            .collect();
4546        let response = ClientControlResponse::CatalogList {
4547            generation,
4548            modules: entries,
4549            subc_ops: subc_ops(),
4550        };
4551        Ok(vec![control_response_body_frame(
4552            &frame,
4553            &response,
4554            "ClientControlResponse::CatalogList",
4555        )?])
4556    }
4557
4558    /// The principal a route.open's `consumer_identity` attests: `Direct`
4559    /// when it presents none, `Reserved` when its launch nonce verifies, and
4560    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4561    /// can run before any refusal check without changing which check refuses.
4562    fn attest_route_open_consumer(
4563        &self,
4564        consumer_identity: Option<&ConsumerIdentity>,
4565    ) -> Option<Principal> {
4566        let Some(consumer_identity) = consumer_identity else {
4567            return Some(Principal::Direct);
4568        };
4569        self.supervisor
4570            .spawned_consumer_authorized(
4571                &consumer_identity.module_id,
4572                &consumer_identity.launch_nonce,
4573            )
4574            .then(|| Principal::Reserved {
4575                module_id: consumer_identity.module_id.clone(),
4576            })
4577    }
4578
4579    /// The caller fields for a refusal that only has the raw request frame,
4580    /// such as the connection-level admission refusal sent before the request
4581    /// reaches [`Self::handle_route_open`].
4582    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4583        let Ok(ClientControlRequest::RouteOpen {
4584            identity,
4585            consumer_identity,
4586            scope,
4587            ..
4588        }) = parse_client_control_request(&frame.body)
4589        else {
4590            return RouteOpenCaller::unknown();
4591        };
4592        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4593        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4594    }
4595
4596    fn route_open_principal(
4597        &self,
4598        frame: &Frame,
4599        consumer_identity: Option<ConsumerIdentity>,
4600    ) -> Result<Result<Principal, Frame>, RouterError> {
4601        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4602        self.route_open_principal_attested(frame, consumer_identity, attested)
4603    }
4604
4605    /// [`Self::route_open_principal`] with `attested` already taken by
4606    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4607    /// open checks the launch nonce once.
4608    fn route_open_principal_attested(
4609        &self,
4610        frame: &Frame,
4611        consumer_identity: Option<ConsumerIdentity>,
4612        attested: Option<Principal>,
4613    ) -> Result<Result<Principal, Frame>, RouterError> {
4614        if let Some(principal) = attested {
4615            return Ok(Ok(principal));
4616        }
4617        let Some(consumer_identity) = consumer_identity else {
4618            return Ok(Ok(Principal::Direct));
4619        };
4620
4621        Ok(Err(control_error_frame(
4622            frame,
4623            "bad_consumer_identity",
4624            format!(
4625                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4626                consumer_identity.module_id
4627            ),
4628        )?))
4629    }
4630
4631    /// Ordinary `route.open` refusals go through here; admission and breaker
4632    /// refusals log separately with their capacity or breaker state. The daemon can
4633    /// attest which code it sent: without the event, a client's "the daemon
4634    /// refused me" and the daemon's own view could only be reconciled by
4635    /// argument. Malformed input (`invalid_project_root`) does not come here;
4636    /// rejecting a request that was never a valid open is not a refusal of one.
4637    #[allow(clippy::too_many_arguments)]
4638    fn route_open_refusal_frame(
4639        &self,
4640        ctx: &RouteCtx,
4641        caller: &RouteOpenCaller,
4642        frame: &Frame,
4643        module_id: &str,
4644        reason: &'static str,
4645        code: &'static str,
4646        message: impl Into<String>,
4647    ) -> Result<Frame, RouterError> {
4648        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4649        control_error_frame(frame, code, message.into())
4650    }
4651
4652    /// Answer a waiting `route.open` whose relayed bind was refused.
4653    ///
4654    /// `daemon_authored` is provenance, not a code match: the daemon
4655    /// synthesized the refusal itself (a drain, a late reply, a scope that
4656    /// ended mid-bind) or the module refused the bind with its own body. A
4657    /// daemon refusal carries the daemon-origin bit; a module's own
4658    /// rejection is relayed with the bit clear, so a module that refuses
4659    /// with `module_reloading` still cannot mint a daemon refusal.
4660    #[allow(clippy::too_many_arguments)]
4661    fn route_bind_rejection(
4662        &self,
4663        ctx: &RouteCtx,
4664        caller: &RouteOpenCaller,
4665        frame: &Frame,
4666        target_module_id: &str,
4667        body: ErrorBody,
4668        daemon_authored: bool,
4669    ) -> Result<Vec<Frame>, RouterError> {
4670        // The daemon's own commit re-check refused the bind because the
4671        // scope ended or changed after admission. The module accepted;
4672        // counting it as a module rejection would blame the module. This
4673        // branch only runs for daemon-synthesized bodies: a module that
4674        // refuses with a scope code is answered as a module rejection below.
4675        if daemon_authored {
4676            let scope_code = match body.code.as_str() {
4677                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4678                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4679                _ => None,
4680            };
4681            if let Some(code) = scope_code {
4682                self.observe_route_open_refusal(
4683                    ctx,
4684                    caller,
4685                    target_module_id,
4686                    "scope_changed_before_commit",
4687                    code,
4688                );
4689                return Ok(vec![control_error_body_frame(frame, body)?]);
4690            }
4691        }
4692        self.counters
4693            .increment_route_open_refused("module_rejected");
4694        info!(
4695            target: "control",
4696            code = "module_rejected",
4697            module_code = ?body.code,
4698            module_id = ?target_module_id,
4699            connection_id = ctx.connection_id.get(),
4700            principal = %caller.principal,
4701            scope = %caller.scope,
4702            harness = %caller.harness,
4703            "route.open refused"
4704        );
4705        let refusal = if daemon_authored {
4706            control_error_body_frame(frame, body)?
4707        } else {
4708            control_relayed_error_body_frame(frame, body)?
4709        };
4710        Ok(vec![refusal])
4711    }
4712
4713    /// Refuse a `route.open` because the target module's bind-relay breaker is
4714    /// open, without attempting the relay.
4715    ///
4716    /// The wire code is `module_timeout`, which is the truth (the module has
4717    /// not been answering binds) and which both SDKs already classify as
4718    /// retryable with capped backoff. Reusing it is what keeps this change out
4719    /// of both SDKs; the daemon-side distinction lives in the counter key
4720    /// instead.
4721    ///
4722    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4723    /// While a breaker is open this fires on every open to that module, and the
4724    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4725    /// produced 261 lines about a single module inside 3000 lines of daemon
4726    /// log. The rare transitions are logged at warn/info instead and the volume
4727    /// is carried by the counter, so the evidence survives without the flood.
4728    /// The debug line keeps a per-refusal record reachable for whoever turns
4729    /// the level up.
4730    #[allow(clippy::too_many_arguments)]
4731    fn route_open_breaker_refusal_frame(
4732        &self,
4733        ctx: &RouteCtx,
4734        caller: &RouteOpenCaller,
4735        frame: &Frame,
4736        module_id: &str,
4737        consecutive_timeouts: u32,
4738        retry_in: Duration,
4739        probe_in_flight: bool,
4740    ) -> Result<Frame, RouterError> {
4741        self.counters
4742            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4743        debug!(
4744            target: "control",
4745            code = "module_timeout",
4746            module_id = ?module_id,
4747            connection_id = ctx.connection_id.get(),
4748            principal = %caller.principal,
4749            scope = %caller.scope,
4750            harness = %caller.harness,
4751            consecutive_timeouts,
4752            retry_in_ms = retry_in.as_millis() as u64,
4753            probe_in_flight,
4754            "route.open refused by open bind-relay breaker"
4755        );
4756        // Say what a caller can act on. An open bind-relay breaker means the
4757        // module timed out accepting several new routes in a row. The module
4758        // is still running and its established routes keep working; only new
4759        // route.open requests are refused until the cooldown ends and one
4760        // test route (the probe) gets through. A message that only counts
4761        // failed relays reads as "the module is down" to a worker that sees it.
4762        let detail = if probe_in_flight {
4763            "one test route is already being tried; retry once it settles".to_string()
4764        } else {
4765            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4766        };
4767        control_error_frame(
4768            frame,
4769            "module_timeout",
4770            format!(
4771                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4772                 timed out in a row); {detail}; its established routes are unaffected"
4773            ),
4774        )
4775    }
4776
4777    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4778    /// requester's bytes (an unknown target is whatever the client sent) and
4779    /// is Debug-formatted so control characters land in the log escaped
4780    /// rather than as terminal sequences for whoever tails it.
4781    ///
4782    /// `reason` names the check that refused, because one wire code has
4783    /// several senders: after a module registers, `target_unavailable` can
4784    /// come from a missing role, an inactive registration, a supervisor that
4785    /// has not marked the process live, a missing forwarding connection, or a
4786    /// failed relay, and a log that records only the code cannot say which of
4787    /// them fired. It is a static, daemon-chosen label per branch, so it is
4788    /// safe to print plainly and stays a closed set.
4789    fn observe_route_open_refusal(
4790        &self,
4791        ctx: &RouteCtx,
4792        caller: &RouteOpenCaller,
4793        module_id: &str,
4794        reason: &'static str,
4795        code: &'static str,
4796    ) {
4797        self.counters.increment_route_open_refused(code);
4798        info!(
4799            target: "control",
4800            code,
4801            reason,
4802            module_id = ?module_id,
4803            connection_id = ctx.connection_id.get(),
4804            principal = %caller.principal,
4805            scope = %caller.scope,
4806            harness = %caller.harness,
4807            "route.open refused"
4808        );
4809        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4810            self.route_outages.record_not_serving(module_id, reason);
4811        }
4812    }
4813
4814    /// Record an ACCEPTED route.open.
4815    ///
4816    /// Refusals have been logged and counted since the attestation work; accepts
4817    /// were invisible, so the daemon knew every principal it stamped and wrote
4818    /// none of them down. The party that attests the identity was the only party
4819    /// not recording it, which left a credential vault unable to name the sender
4820    /// of a call that reached it (claustrum #43) and left the launch-nonce
4821    /// concurrency question unanswerable from the outside.
4822    ///
4823    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4824    /// `code`/`module_id`/`connection_id` returns both directions of the same
4825    /// decision rather than two shapes a reader has to join by hand.
4826    ///
4827    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4828    /// and the difference carries information rather than being an
4829    /// inconsistency. This line is only reachable after a successful bind to a
4830    /// REGISTERED module, so the value has already passed HELLO validation
4831    /// including the path-hazard refusal and cannot contain control bytes. A
4832    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4833    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4834    ///
4835    /// Bare is also what every other daemon line already emits (`module
4836    /// registered`, `configured module supervised`). Shipping `?module_id` here
4837    /// made this instrument the only one in the file whose ids did not answer
4838    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4839    /// line whose whole purpose is being grepped beside its sibling.
4840    ///
4841    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4842    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4843    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4844    /// one are recorded identically. A test written against that harness passes
4845    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4846    /// green assertion that cannot fail. The same limit applies to the escaping
4847    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4848    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4849    /// Fencing either needs the real formatter, not the capture layer.
4850    ///
4851    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4852    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4853    /// to record. The ephemeral port would decay within minutes and answer only a
4854    /// live question. The identity question is instead answered by counting
4855    /// distinct live connections presenting one module's `consumer_identity` --
4856    /// "is anyone else holding this secret" rather than "is this the right
4857    /// process".
4858    fn observe_route_open_accept(
4859        &self,
4860        ctx: &RouteCtx,
4861        module_id: &str,
4862        principal: &str,
4863        scope: &str,
4864    ) {
4865        self.route_outages.record_accepted(module_id);
4866        self.counters.increment_route_open_accepted(principal);
4867        info!(
4868            target: "control",
4869            principal,
4870            module_id,
4871            connection_id = ctx.connection_id.get(),
4872            scope = %scope,
4873            "route.open accepted"
4874        );
4875    }
4876
4877    fn supervised_absent_route_open_refusal_frame(
4878        &self,
4879        ctx: &RouteCtx,
4880        caller: &RouteOpenCaller,
4881        frame: &Frame,
4882        module_id: &str,
4883        code: &'static str,
4884        status: &crate::supervise::ModuleStatus,
4885    ) -> Result<Frame, RouterError> {
4886        self.counters.increment_route_open_refused(code);
4887        info!(
4888            target: "control",
4889            code,
4890            reason = "supervised_not_registered",
4891            module_id = ?module_id,
4892            connection_id = ctx.connection_id.get(),
4893            principal = %caller.principal,
4894            scope = %caller.scope,
4895            harness = %caller.harness,
4896            state = %status.state,
4897            enabled = status.enabled,
4898            live = status.live,
4899            "route.open refused"
4900        );
4901        // A supervised module whose process has not registered is not
4902        // serving, whatever the reason; the supervisor knows this id, so it is
4903        // safe to track.
4904        self.route_outages
4905            .record_not_serving(module_id, "supervised_not_registered");
4906        let message = if let Some(pid) = status.waiting_for_previous_pid {
4907            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4908                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4909            })
4910        } else {
4911            format!(
4912                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4913                status.state, status.enabled, status.live
4914            )
4915        };
4916        control_error_frame(frame, code, message)
4917    }
4918
4919    async fn handle_route_open(
4920        &self,
4921        ctx: &RouteCtx,
4922        frame: Frame,
4923        request: RouteOpenRequest,
4924        timings: &mut RouteOpenTimings,
4925    ) -> Result<Vec<Frame>, RouterError> {
4926        let RouteOpenRequest {
4927            target,
4928            mut identity,
4929            consumer_identity,
4930            consumer_capabilities,
4931            role_versions,
4932            busy_refusals,
4933            admission_facts,
4934            scope,
4935        } = request;
4936        let target_module_id = target_module_id(&target).to_string();
4937        timings.module_id.clone_from(&target_module_id);
4938        if self
4939            .registry
4940            .get_module_by_connection(ctx.connection_id)
4941            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4942            .is_some()
4943        {
4944            return Ok(vec![control_error_frame(
4945                &frame,
4946                "invalid_request",
4947                "module connections cannot open client routes",
4948            )?]);
4949        }
4950        debug!(
4951            connection_id = ctx.connection_id.get(),
4952            corr = frame.header.corr,
4953            module_id = %target_module_id,
4954            "handling route.open"
4955        );
4956
4957        // The caller's attested principal, requested scope and harness, for
4958        // every `route.open refused` line below. The launch nonce is checked
4959        // here, once, and the answer is reused where the principal is
4960        // enforced further down, so the order of refusals is unchanged.
4961        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4962        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4963
4964        // A malformed declaration is refused first, before anything about the
4965        // target is looked up: the same body would be refused against any
4966        // module, so the caller learns nothing by retrying or waiting. An empty
4967        // map declares nothing and travels as no field at all, so a provider
4968        // only ever sees a missing field or a non-empty one.
4969        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4970        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4971            self.observe_route_open_refusal(
4972                ctx,
4973                &caller,
4974                &target_module_id,
4975                "invalid_role_versions",
4976                error_codes::INVALID_REQUEST,
4977            );
4978            return Ok(vec![control_error_body_frame(
4979                &frame,
4980                ErrorBody {
4981                    code: error_codes::INVALID_REQUEST.to_string(),
4982                    message: error.to_string(),
4983                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4984                },
4985            )?]);
4986        }
4987
4988        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4989            if status.waiting_for_previous_pid.is_some() {
4990                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4991                    ctx,
4992                    &caller,
4993                    &frame,
4994                    &target_module_id,
4995                    "module_warming",
4996                    &status,
4997                )?]);
4998            }
4999        }
5000
5001        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
5002        // opposite. Below, a caller learns whether a module is unregistered,
5003        // supervised-but-down (with state/enabled/live), or registered without the
5004        // requested role. Elsewhere that is an enumeration leak: a probe learning
5005        // the shape of a fleet it cannot otherwise see.
5006        //
5007        // It is not one here, and the reason is the ACCESS MODEL rather than
5008        // anything about these errors. Reaching route.open requires the
5009        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
5010        // connection file, so any caller who completes it already runs as this
5011        // user -- and can read subc.jsonc for the module list and `ck module
5012        // status` for live state. The reply discloses nothing the caller cannot
5013        // read more easily from disk, while the precision is load-bearing:
5014        // `unknown_module` is retryable and a missing role is not.
5015        //
5016        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
5017        // remote transport, a sandboxed caller, a shared-host mode -- THAT
5018        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
5019        let Some(registration) = self
5020            .registry
5021            .get_module(&target_module_id)
5022            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5023        else {
5024            if let Some((status, warming)) =
5025                self.supervisor_status(&target_module_id, frame.header.corr)?
5026            {
5027                // BEFORE the two availability codes below, because for a module
5028                // that speaks no subc wire both of them are false comfort: they
5029                // say "not right now" and are retried, and this module will
5030                // never register no matter how long the caller waits. The
5031                // absence here is the declaration being honoured, not a module
5032                // that is late.
5033                if status.protocol == ModuleProtocol::None {
5034                    return Ok(vec![self.route_open_refusal_frame(
5035                        ctx,
5036                        &caller,
5037                        &frame,
5038                        &target_module_id,
5039                        "protocol_none",
5040                        error_codes::MODULE_NO_PROTOCOL,
5041                        format!(
5042                            "module_id '{target_module_id}' is declared protocol: none; \
5043                             it speaks no subc wire and serves no routes"
5044                        ),
5045                    )?]);
5046                }
5047                let code = if warming {
5048                    "module_warming"
5049                } else {
5050                    "target_unavailable"
5051                };
5052                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5053                    ctx,
5054                    &caller,
5055                    &frame,
5056                    &target_module_id,
5057                    code,
5058                    &status,
5059                )?]);
5060            }
5061            if let Some(removed_ago_ms) =
5062                self.supervisor.removal_tombstone_age_ms(&target_module_id)
5063            {
5064                return Ok(vec![self.route_open_refusal_frame(
5065                    ctx,
5066                    &caller,
5067                    &frame,
5068                    &target_module_id,
5069                    "removed",
5070                    error_codes::MODULE_REMOVED,
5071                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5072                )?]);
5073            }
5074            return Ok(vec![self.route_open_refusal_frame(
5075                ctx,
5076                &caller,
5077                &frame,
5078                &target_module_id,
5079                "not_registered",
5080                error_codes::UNKNOWN_MODULE,
5081                format!("module_id '{target_module_id}' is not registered"),
5082            )?]);
5083        };
5084
5085        // Best-effort only: registry readiness and forwarding reservation use
5086        // different locks, so a module can flip readiness between this read and
5087        // the relay. Modules must still tolerate an `on_bind` while not ready.
5088        if !registration.ready {
5089            self.counters
5090                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5091            info!(
5092                target: "control",
5093                code = error_codes::MODULE_WARMING,
5094                module_id = ?target_module_id,
5095                connection_id = ctx.connection_id.get(),
5096                principal = %caller.principal,
5097                scope = %caller.scope,
5098                harness = %caller.harness,
5099                reason = "declared_not_ready",
5100                "route.open refused"
5101            );
5102            // The module is registered but says it cannot take work, which is
5103            // an outage from the caller's side even though its process is up.
5104            self.route_outages
5105                .record_not_serving(&target_module_id, "declared_not_ready");
5106            return Ok(vec![control_error_body_frame(
5107                &frame,
5108                ErrorBody {
5109                    code: error_codes::MODULE_WARMING.to_string(),
5110                    message: format!(
5111                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5112                    ),
5113                    detail: Some(serde_json::json!({
5114                        "reason": "declared_not_ready"
5115                    })),
5116                },
5117            )?]);
5118        }
5119
5120        // Effective readiness, second half: a module that declares a capability
5121        // `need: required` is not routable while that capability has no
5122        // registered provider. It is enforced HERE, as a retryable routing
5123        // refusal, and deliberately not as spawn ordering or a boot block. The
5124        // module is still started and registered and can make its own calls;
5125        // spawn ordering is a promise that cannot be kept once a provider
5126        // crashes at runtime, and refusing to boot would stop the whole
5127        // machine, including the tools needed to fix its configuration.
5128        //
5129        // "Provided" is the evaluator's verdict, which counts a provider as
5130        // soon as it has REGISTERED, not once it is ready. Two modules that
5131        // require each other's capabilities are therefore both routable once
5132        // both register; counting readiness instead would deadlock them.
5133        //
5134        // Only new opens are refused. Routes already bound when a provider
5135        // goes away stay bound: nothing here tears them down, and the module
5136        // answers them as it can. Like the readiness read above this is
5137        // best-effort against a provider registering or leaving concurrently.
5138        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
5139            self.counters
5140                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5141            info!(
5142                target: "control",
5143                code = error_codes::MODULE_WARMING,
5144                module_id = ?target_module_id,
5145                connection_id = ctx.connection_id.get(),
5146                principal = %caller.principal,
5147                scope = %caller.scope,
5148                harness = %caller.harness,
5149                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5150                capability = %capability,
5151                "route.open refused"
5152            );
5153            return Ok(vec![control_error_body_frame(
5154                &frame,
5155                ErrorBody {
5156                    code: error_codes::MODULE_WARMING.to_string(),
5157                    message: format!(
5158                        "module_id '{target_module_id}' requires capability '{capability}', \
5159                         which no registered module provides; retry"
5160                    ),
5161                    detail: Some(serde_json::json!({
5162                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5163                        "capability": capability,
5164                    })),
5165                },
5166            )?]);
5167        }
5168
5169        if !target_has_required_role(&target, &registration.manifest.provides) {
5170            return Ok(vec![self.route_open_refusal_frame(
5171                ctx,
5172                &caller,
5173                &frame,
5174                &target_module_id,
5175                "role_not_provided",
5176                "target_unavailable",
5177                format!("module_id '{target_module_id}' does not provide the requested target"),
5178            )?]);
5179        }
5180
5181        if registration.state != ChannelState::Active {
5182            return Ok(vec![self.route_open_refusal_frame(
5183                ctx,
5184                &caller,
5185                &frame,
5186                &target_module_id,
5187                "registration_not_active",
5188                "target_unavailable",
5189                format!("module_id '{target_module_id}' is not active"),
5190            )?]);
5191        }
5192
5193        if self
5194            .forwarding
5195            .module_is_draining(&target_module_id)
5196            .map_err(RouterError::Forwarding)?
5197        {
5198            return Ok(vec![self.route_open_refusal_frame(
5199                ctx,
5200                &caller,
5201                &frame,
5202                &target_module_id,
5203                "reloading",
5204                "module_reloading",
5205                format!("module_id '{target_module_id}' is reloading"),
5206            )?]);
5207        }
5208
5209        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5210            process_liveness.process_live(&target_module_id) == Some(false)
5211        }) {
5212            // A module the supervisor is restarting or reloading can still hold
5213            // a registration: the old process before its connection closes, or
5214            // a new one that registered while the supervisor was draining. The
5215            // forwarding table does not see that as draining, but the consumer
5216            // should still be told to retry soon, exactly as for the drain
5217            // above, rather than that the target is unavailable.
5218            if process_liveness.process_replacing(&target_module_id) {
5219                return Ok(vec![self.route_open_refusal_frame(
5220                    ctx,
5221                    &caller,
5222                    &frame,
5223                    &target_module_id,
5224                    "reloading",
5225                    "module_reloading",
5226                    format!("module_id '{target_module_id}' is reloading"),
5227                )?]);
5228            }
5229            return Ok(vec![self.route_open_refusal_frame(
5230                ctx,
5231                &caller,
5232                &frame,
5233                &target_module_id,
5234                "supervisor_not_live",
5235                "target_unavailable",
5236                format!("module_id '{target_module_id}' is not live"),
5237            )?]);
5238        }
5239
5240        if !self
5241            .forwarding
5242            .has_live_module_connection(&target_module_id)
5243            .map_err(RouterError::Forwarding)?
5244        {
5245            return Ok(vec![self.route_open_refusal_frame(
5246                ctx,
5247                &caller,
5248                &frame,
5249                &target_module_id,
5250                "no_forwarding_connection",
5251                "target_unavailable",
5252                format!("module_id '{target_module_id}' has no live forwarding connection"),
5253            )?]);
5254        }
5255
5256        if let Some(error) =
5257            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5258        {
5259            self.observe_route_open_refusal(
5260                ctx,
5261                &caller,
5262                &target_module_id,
5263                "op_not_allowed",
5264                "op_not_allowed",
5265            );
5266            return Ok(vec![error]);
5267        }
5268
5269        let principal =
5270            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5271                Ok(principal) => principal,
5272                Err(error) => {
5273                    self.observe_route_open_refusal(
5274                        ctx,
5275                        &caller,
5276                        &target_module_id,
5277                        "bad_consumer_identity",
5278                        "bad_consumer_identity",
5279                    );
5280                    return Ok(vec![error]);
5281                }
5282            };
5283        timings.principal = principal_label(&principal);
5284
5285        // Only Callosum can attest a production phone build. Consume the claim
5286        // here so it can be kept on the route without reaching any module.
5287        let build_class = identity.build_class.take().filter(|build_class| {
5288            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5289                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5290            } else {
5291                if self
5292                    .build_class_warned
5293                    .lock()
5294                    .unwrap_or_else(|p| p.into_inner())
5295                    .insert(ctx.connection_id)
5296                {
5297                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5298                        "ignoring build_class from a non-Callosum opener");
5299                }
5300                false
5301            }
5302        });
5303
5304        // This is attested, control-plane policy for supervised module origins.
5305        // Keep it before route reservation and out of the opaque forwarding hot
5306        // path: data frames must never acquire a per-frame capability check.
5307        if let Principal::Reserved {
5308            module_id: opening_module_id,
5309        } = &principal
5310        {
5311            if let Some(opening_registration) = self
5312                .registry
5313                .get_module(opening_module_id)
5314                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5315            {
5316                if let Some(capability) =
5317                    denied_capability(&opening_registration.manifest, &registration.manifest)
5318                {
5319                    warn!(
5320                        opening_module_id,
5321                        target_module_id,
5322                        capability,
5323                        "refusing route.open because an attested capability deny edge matches"
5324                    );
5325                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5326                        &frame,
5327                        &target_module_id,
5328                        "capability_deny_edge",
5329                        "capability_forbidden",
5330                        format!(
5331                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5332                        ),
5333                    )?]);
5334                }
5335            }
5336        }
5337
5338        if admission_facts.is_some() {
5339            let carrier_matches = matches!(
5340                &principal,
5341                Principal::Reserved { module_id }
5342                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5343            );
5344            if !carrier_matches {
5345                return Ok(vec![self.route_open_refusal_frame(
5346                    ctx,
5347                    &caller,
5348                    &frame,
5349                    &target_module_id,
5350                    "admission_facts_carrier_not_permitted",
5351                    "admission_facts_not_permitted",
5352                    "admission facts may only be carried by the configured reserved module",
5353                )?]);
5354            }
5355
5356            let target_allowed = self
5357                .admission_facts_targets
5358                .as_ref()
5359                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5360            if !target_allowed {
5361                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5362                    &frame,
5363                    &target_module_id,
5364                    "admission_facts_target_not_listed",
5365                    "admission_facts_target_not_allowed",
5366                    format!(
5367                        "admission facts are not permitted for target module_id '{target_module_id}'"
5368                    ),
5369                )?]);
5370            }
5371
5372            // Keep the value opaque to subc. The downstream admission validator owns
5373            // schema and semantic checks; this daemon only enforces carrier authority
5374            // and the configured destination allowlist.
5375        }
5376
5377        // Scope admission, on the attested principal above and never on the
5378        // request body. The tag read here travels with the pending bind and is
5379        // compared with the published one at commit, so a sync between here
5380        // and the module's ack refuses the open instead of binding a stamp
5381        // that is no longer true.
5382        let (bound_scope, mut scope_stamp) = match &scope {
5383            None => (None, None),
5384            Some(selector) => {
5385                let owner_configured = match &selector.owner {
5386                    // A reserved owner counts as configured from before its
5387                    // process is spawned (see `SupervisorHandle::is_configured`).
5388                    // So an owner that has not synced its scopes yet is refused
5389                    // as retryable (`scope_not_synced`), not as one that will
5390                    // never sync.
5391                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5392                    _ => false,
5393                };
5394                let mut scopes = self
5395                    .scopes
5396                    .write()
5397                    .unwrap_or_else(|poisoned| poisoned.into_inner());
5398                let admitted = scopes
5399                    .admit(&principal, &target_module_id, selector, owner_configured)
5400                    .and_then(|admission| {
5401                        crate::scopes::check_target_flow_support(
5402                            &admission.stamp,
5403                            &target_module_id,
5404                            registration.manifest.capabilities.as_ref(),
5405                        )?;
5406                        crate::scopes::check_target_agent_run_support(
5407                            &admission.stamp,
5408                            &target_module_id,
5409                            registration.manifest.capabilities.as_ref(),
5410                        )?;
5411                        let bound = scopes.reserve_bind(
5412                            &admission,
5413                            &principal,
5414                            ctx.connection_id,
5415                            selector.hold_presence,
5416                        );
5417                        Ok((admission, bound))
5418                    });
5419                match admitted {
5420                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5421                    Err(refusal) => {
5422                        return Ok(vec![self.route_open_refusal_frame(
5423                            ctx,
5424                            &caller,
5425                            &frame,
5426                            &target_module_id,
5427                            refusal.code,
5428                            refusal.code,
5429                            refusal.message,
5430                        )?]);
5431                    }
5432                }
5433            }
5434        };
5435        // From here on a refusal names the scope by the stamp admission
5436        // issued, exactly as the `route.open accepted` line does.
5437        if let Some(stamp) = &scope_stamp {
5438            caller.scope = route_open_scope_label(stamp);
5439        }
5440
5441        // Allow resolvable missing-tail paths so a vanished directory does not
5442        // prevent reopening an existing session for cleanup. Binding does not
5443        // authorize creation of durable provider state.
5444        //
5445        // Resolve real paths rather than lexical cleanup: canonicalize the
5446        // longest existing ancestor and append the missing tail, retaining the
5447        // path's real existing ancestry even when a leaf is missing.
5448        timings.enter(RouteOpenPhase::ProjectRoot);
5449        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5450            Ok(project) => project,
5451            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5452        };
5453        timings.enter(RouteOpenPhase::Admission);
5454        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5455            Ok(root) => root,
5456            Err(err) => {
5457                return Ok(vec![control_error_frame(
5458                    &frame,
5459                    "invalid_project_root",
5460                    err.to_string(),
5461                )?])
5462            }
5463        };
5464        // Compute compatibility project fields before stripping optional stamp
5465        // members. Preserve the canonical opener root and its project ID,
5466        // including absence. For rootless openers, use stamp workdir or `/`, and
5467        // opener ID, else stamp ID, else no ID. Emit these identity fields only
5468        // to targets without bind-no-project/v1.
5469        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5470
5471        // Last gate before any relay work, and deliberately after the cheap
5472        // registry and availability checks above: those name a more precise
5473        // condition (unknown, removed, reloading) and a caller is better served
5474        // by the precise code than by this one.
5475        //
5476        // Everything below this point costs an egress permit, a reserved handle
5477        // pair and, if the module does not answer, the whole relay budget. The
5478        // reader no longer waits for that budget, so cap each target explicitly;
5479        // serial dispatch used to provide the accidental cap of one relay per
5480        // connection. Admission is a mutex-protected count and never waits.
5481        let _concurrency_guard = match self
5482            .route_bind_concurrency
5483            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5484        {
5485            Ok(guard) => guard,
5486            Err(in_flight) => {
5487                return Ok(vec![self.route_open_target_capacity_refusal(
5488                    ctx,
5489                    &caller,
5490                    &frame,
5491                    &target_module_id,
5492                    in_flight,
5493                )?]);
5494            }
5495        };
5496
5497        // A module that has already burned the whole budget `threshold` times
5498        // in a row does not get to charge it again until a probe says it recovered.
5499        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5500            RouteBindAdmission::Admitted { guard, probe } => {
5501                if probe {
5502                    info!(
5503                        module_id = %target_module_id,
5504                        connection_id = ctx.connection_id.get(),
5505                        "route.bind breaker half-open: admitting one probe"
5506                    );
5507                }
5508                guard
5509            }
5510            RouteBindAdmission::Refused {
5511                consecutive_timeouts,
5512                retry_in,
5513                probe_in_flight,
5514            } => {
5515                return Ok(vec![self.route_open_breaker_refusal_frame(
5516                    ctx,
5517                    &caller,
5518                    &frame,
5519                    &target_module_id,
5520                    consecutive_timeouts,
5521                    retry_in,
5522                    probe_in_flight,
5523                )?]);
5524            }
5525        };
5526
5527        // Resolve the per-module budget here so the wait matches the operator's
5528        // intent for this specific target. A per-module override in
5529        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5530        // daemons) wins over the daemon-wide default.
5531        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5532        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5533        let operator_app = self
5534            .operator_apps
5535            .lock()
5536            .unwrap_or_else(|p| p.into_inner())
5537            .get(&ctx.connection_id)
5538            .cloned();
5539        timings.enter(RouteOpenPhase::EgressReservation);
5540        let pending = match self
5541            .forwarding
5542            .begin_route_bind_relay_for(
5543                ctx.connection_id,
5544                ctx.egress.clone(),
5545                response_version(&frame),
5546                frame.header.corr,
5547                &target_module_id,
5548                principal.clone(),
5549                bound_scope,
5550                project_root,
5551                identity.harness.clone(),
5552                build_class,
5553                operator_app.clone(),
5554                relay_deadline,
5555                busy_refusals == Some(true),
5556            )
5557            .await
5558        {
5559            Ok(pending) => pending,
5560            Err(err) => {
5561                return Ok(vec![self.route_open_refusal_frame(
5562                    ctx,
5563                    &caller,
5564                    &frame,
5565                    &target_module_id,
5566                    "relay_reservation_failed",
5567                    forwarding_error_code(&err),
5568                    err.to_string(),
5569                )?])
5570            }
5571        };
5572        timings.enter(RouteOpenPhase::Admission);
5573        let crate::forwarding::PendingRouteBindRelay {
5574            endpoint,
5575            module_sink,
5576            negotiated_ver,
5577            client_channel,
5578            client_epoch,
5579            module_channel,
5580            module_epoch,
5581            corr: relay_corr,
5582            receiver,
5583        } = pending;
5584        let mut reservation =
5585            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5586
5587        // Reserving egress can wait while a module reconnects or a swap cuts
5588        // over. Check the relay's captured connection, not the earlier by-id
5589        // lookup: the original module's flow/run capabilities cannot authorize
5590        // its replacement. The captured sink stays bound to that connection.
5591        let relay_registration = self
5592            .registry
5593            .get_module_by_connection(endpoint.connection_id)
5594            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5595        let relay_capabilities = relay_registration
5596            .as_ref()
5597            .and_then(|registration| registration.manifest.capabilities.as_ref());
5598        if let Some(selector) = &scope {
5599            // Egress reservation may wait across a scope update. Recheck the
5600            // proof against the current record before sending the captured bind.
5601            if let Err(refusal) = self
5602                .scopes
5603                .read()
5604                .unwrap_or_else(|poisoned| poisoned.into_inner())
5605                .check_bind_proof(&principal, selector)
5606            {
5607                reservation.release_and_disarm();
5608                return Ok(vec![self.route_open_refusal_frame(
5609                    ctx,
5610                    &caller,
5611                    &frame,
5612                    &target_module_id,
5613                    refusal.code,
5614                    refusal.code,
5615                    refusal.message,
5616                )?]);
5617            }
5618        }
5619        if let Some(stamp) = scope_stamp.as_ref() {
5620            if let Err(refusal) = crate::scopes::check_target_flow_support(
5621                stamp,
5622                &target_module_id,
5623                relay_capabilities,
5624            )
5625            .and_then(|()| {
5626                crate::scopes::check_target_agent_run_support(
5627                    stamp,
5628                    &target_module_id,
5629                    relay_capabilities,
5630                )
5631            }) {
5632                reservation.release_and_disarm();
5633                return Ok(vec![self.route_open_refusal_frame(
5634                    ctx,
5635                    &caller,
5636                    &frame,
5637                    &target_module_id,
5638                    refusal.code,
5639                    refusal.code,
5640                    refusal.message,
5641                )?]);
5642            }
5643        }
5644
5645        // Older readers may reject unknown fields. Select optional fields from
5646        // the captured endpoint's declaration, including on unscoped binds.
5647        if scope_stamp
5648            .as_ref()
5649            .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5650            && !crate::scopes::target_supports_federated(relay_capabilities)
5651        {
5652            // Keep bound_scope: even an unstamped legacy route belongs to this epoch.
5653            scope_stamp = None;
5654        }
5655        if !relay_capabilities.is_some_and(|caps| {
5656            caps.provides
5657                .iter()
5658                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5659        }) {
5660            if let Some(stamp) = scope_stamp.as_mut() {
5661                stamp.workdir = None;
5662                stamp.project_id = None;
5663            }
5664        }
5665        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5666        // attribution. The run gate above has already required the target to
5667        // declare `worker-runs/v1`, which promises to read both
5668        // `on_behalf_of_agent` and `call_mode`.
5669        let worker_run = scope_stamp
5670            .as_ref()
5671            .is_some_and(crate::scopes::is_worker_run)
5672            && relay_capabilities.is_some_and(|caps| {
5673                caps.provides
5674                    .iter()
5675                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5676            });
5677        if !worker_run
5678            && !relay_capabilities.is_some_and(|caps| {
5679                caps.provides
5680                    .iter()
5681                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5682            })
5683        {
5684            if let Some(stamp) = scope_stamp.as_mut() {
5685                stamp.attributes.on_behalf_of_agent = None;
5686            }
5687        }
5688        if !relay_capabilities.is_some_and(|caps| {
5689            caps.provides
5690                .iter()
5691                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5692        }) {
5693            if let Some(stamp) = scope_stamp.as_mut() {
5694                stamp.attributes.flow_version = None;
5695                stamp.attributes.flow_code_hash = None;
5696                stamp.attributes.flow_install_id = None;
5697            }
5698        }
5699        let call_mode = if worker_run
5700            || relay_capabilities
5701                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5702        {
5703            Some(derive_call_mode(scope_stamp.as_ref()))
5704        } else {
5705            if let Some(stamp) = scope_stamp.as_mut() {
5706                stamp.expires_at_ms = None;
5707            }
5708            None
5709        };
5710
5711        debug!(
5712            connection_id = ctx.connection_id.get(),
5713            client_channel,
5714            client_epoch,
5715            module_channel,
5716            module_epoch,
5717            "reserved route handle pair"
5718        );
5719        // Rendered BEFORE the move into the relay, because the accept arm below
5720        // is where it is logged and the principal is gone by then.
5721        let route_principal_label = principal_label(&principal);
5722        let relay = ModuleControlRequest::RouteBind {
5723            route_channel: module_channel,
5724            epoch: module_epoch,
5725            target,
5726            identity,
5727            principal: Some(principal),
5728            consumer_capabilities,
5729            role_versions,
5730            admission_facts,
5731            scope: scope_stamp,
5732            call_mode,
5733        };
5734        let peer = RouteBindPeer::new()
5735            .with_observed_peer(
5736                relay_capabilities
5737                    .is_some_and(|caps| {
5738                        caps.provides
5739                            .iter()
5740                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5741                    })
5742                    .then(|| {
5743                        self.observed_peers
5744                            .lock()
5745                            .unwrap_or_else(|p| p.into_inner())
5746                            .get(&ctx.connection_id)
5747                            .cloned()
5748                            .unwrap_or_else(ObservedPeer::unobserved)
5749                    }),
5750            )
5751            .with_operator_app(
5752                self.connection_operator_app(operator_app.as_ref())
5753                    .filter(|_| {
5754                        relay_capabilities.is_some_and(|caps| {
5755                            caps.provides
5756                                .iter()
5757                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5758                        })
5759                    }),
5760            );
5761        let relay_body = crate::bind_compat::encode_bind(
5762            &relay,
5763            &peer,
5764            &legacy_bind_project,
5765            relay_capabilities,
5766        )
5767        .map_err(|err| {
5768            RouterError::backend(
5769                0,
5770                frame.header.corr,
5771                format!("failed to encode route.bind request: {err}"),
5772            )
5773        })?;
5774        let relay_frame = Frame::build_with_version(
5775            negotiated_ver,
5776            FrameType::Request,
5777            control_flags(),
5778            0,
5779            0,
5780            relay_corr,
5781            relay_body,
5782        )
5783        .map_err(RouterError::FrameBuild)?;
5784
5785        timings.enter(RouteOpenPhase::BindSend);
5786        let sent = module_sink.send(relay_frame).await;
5787        timings.enter(RouteOpenPhase::Admission);
5788        if let Err(err) = sent {
5789            reservation.release_and_disarm();
5790            return Ok(vec![self.route_open_refusal_frame(
5791                ctx,
5792                &caller,
5793                &frame,
5794                &target_module_id,
5795                "relay_send_failed",
5796                "target_unavailable",
5797                err.to_string(),
5798            )?]);
5799        }
5800
5801        #[cfg(test)]
5802        if let Some(hold) = &self.route_bind_relay_sent_hold {
5803            // First wait: tell the test the bind is queued. Second wait: stay
5804            // here until the test has driven whatever it wants to happen
5805            // between queueing the bind and marking it enqueued.
5806            hold.wait().await;
5807            hold.wait().await;
5808        }
5809
5810        // A missing relay is not by itself an abandoned bind: the module may
5811        // already have acked it and the route may be committed. Only a relay
5812        // dropped without any module reply needs this GOODBYE; one a reply
5813        // settled has already had its GOODBYE decided by the reply path.
5814        if self
5815            .forwarding
5816            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5817            .map_err(RouterError::Forwarding)?
5818            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5819        {
5820            self.send_abandoned_route_bind_goodbye(
5821                &module_sink,
5822                negotiated_ver,
5823                module_channel,
5824                module_epoch,
5825            );
5826        }
5827
5828        timings.enter(RouteOpenPhase::BindAckWait);
5829        let outcome = timeout_at(relay_deadline, receiver).await;
5830        timings.enter(RouteOpenPhase::Admission);
5831        match outcome {
5832            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5833                reservation.disarm();
5834                if breaker.record_accepted() {
5835                    info!(
5836                        module_id = %target_module_id,
5837                        "route.bind breaker closed: the probe was accepted"
5838                    );
5839                }
5840                self.observe_route_open_accept(
5841                    ctx,
5842                    &target_module_id,
5843                    &route_principal_label,
5844                    &caller.scope,
5845                );
5846                Ok(Vec::new())
5847            }
5848            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5849                reservation.release_and_disarm();
5850                // A module that says no in microseconds is healthy. Rejection
5851                // is a different condition with its own refusal and must not
5852                // move the breaker.
5853                breaker.record_inconclusive();
5854                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5855            }
5856            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5857                reservation.release_and_disarm();
5858                breaker.record_inconclusive();
5859                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5860            }
5861            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5862                reservation.release_and_disarm();
5863                breaker.record_inconclusive();
5864                // Fires when the module's connection closes while a relayed
5865                // bind is pending -- typically a caller racing a module restart
5866                // whose bind was relayed BEFORE the drain mark went up. Logged
5867                // because the caller sees only its own error and the fleet has
5868                // already spent one diagnosis round unable to tell this arm
5869                // from a relay timeout without daemon-side evidence.
5870                tracing::warn!(
5871                    module_id = %target_module_id,
5872                    "route.bind relay abandoned: {message}"
5873                );
5874                Ok(vec![self.route_open_refusal_frame(
5875                    ctx,
5876                    &caller,
5877                    &frame,
5878                    &target_module_id,
5879                    "relay_abandoned",
5880                    "target_unavailable",
5881                    message,
5882                )?])
5883            }
5884            Ok(Err(_)) => {
5885                reservation.release_and_disarm();
5886                breaker.record_inconclusive();
5887                Ok(vec![self.route_open_refusal_frame(
5888                    ctx,
5889                    &caller,
5890                    &frame,
5891                    &target_module_id,
5892                    "relay_waiter_canceled",
5893                    "target_unavailable",
5894                    "route.bind relay waiter was canceled before the module responded",
5895                )?])
5896            }
5897            Err(_) => {
5898                reservation.release_and_disarm();
5899                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5900                // answer at all is the one condition a fast refusal can
5901                // usefully stand in for; every other arm already answered.
5902                if let Some(opened) = breaker.record_timeout(
5903                    self.route_bind_breaker_threshold,
5904                    self.route_bind_breaker_cooldown,
5905                ) {
5906                    warn!(
5907                        module_id = %target_module_id,
5908                        consecutive_timeouts = opened.consecutive_timeouts,
5909                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5910                        reopened_after_probe = opened.reopened_after_probe,
5911                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5912                    );
5913                }
5914                // The generous budget just burned to no answer: the module is
5915                // registered and its connection is up, but its bind handler sat
5916                // on the ack for the full budget (warm-on-bind, cold configure,
5917                // or a wedged handler). Every earlier unavailability shape
5918                // fast-refuses BEFORE the relay, so this arm firing means the
5919                // slowness is module-side -- log it so the per-module timeline
5920                // is reconstructable without client audit rows.
5921                tracing::warn!(
5922                    module_id = %target_module_id,
5923                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5924                    "route.bind relay timed out: module did not ack within budget"
5925                );
5926                Ok(vec![self.route_open_refusal_frame(
5927                    ctx,
5928                    &caller,
5929                    &frame,
5930                    &target_module_id,
5931                    "relay_timed_out",
5932                    "module_timeout",
5933                    format!(
5934                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5935                        route_bind_relay_timeout
5936                    ),
5937                )?])
5938            }
5939        }
5940    }
5941
5942    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5943        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5944            snapshot: self.supervisor.spawn_snapshot(),
5945        };
5946        Ok(vec![control_response_body_frame(
5947            &frame,
5948            &response,
5949            "ClientControlResponse::SupervisorSpawnSnapshot",
5950        )?])
5951    }
5952
5953    fn handle_supervisor_spawn_subscribe(
5954        &self,
5955        ctx: &RouteCtx,
5956        frame: Frame,
5957        since: Option<SpawnCursor>,
5958    ) -> Result<Vec<Frame>, RouterError> {
5959        match self.supervisor.subscribe_spawns(
5960            ctx.connection_id,
5961            frame.header.corr,
5962            response_version(&frame),
5963            since,
5964            ctx.egress.clone(),
5965        ) {
5966            Ok(()) => Ok(Vec::new()),
5967            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5968                Ok(vec![control_error_body_frame(
5969                    &frame,
5970                    ErrorBody {
5971                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5972                        message: "spawn cursor belongs to a different daemon incarnation"
5973                            .to_string(),
5974                        detail: Some(serde_json::json!({
5975                            "current_daemon_incarnation": current
5976                        })),
5977                    },
5978                )?])
5979            }
5980            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5981                &frame,
5982                ErrorBody {
5983                    code: "spawn_cursor_too_old".to_string(),
5984                    message: "spawn cursor predates the retained event ring".to_string(),
5985                    detail: Some(serde_json::json!({
5986                        "oldest_retained_cursor": oldest
5987                    })),
5988                },
5989            )?]),
5990            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5991                0,
5992                frame.header.corr,
5993                format!("failed to open supervisor spawn subscription: {error}"),
5994            )),
5995        }
5996    }
5997
5998    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5999        // Status/configuration snapshots may wait on synchronous supervisor
6000        // locks. Keep that work off the connection's async runtime thread.
6001        let handler = self.clone();
6002        let runtime = tokio::runtime::Handle::current();
6003        // The blocking pool does not inherit the control request's tracing
6004        // subscriber. Preserve it so slow snapshot-lock warnings stay visible.
6005        let subscriber = tracing::dispatcher::get_default(Clone::clone);
6006        let corr = frame.header.corr;
6007        tokio::task::spawn_blocking(move || {
6008            runtime.block_on(
6009                list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6010                    .with_subscriber(subscriber),
6011            )
6012        })
6013        .await
6014        .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6015    }
6016
6017    async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6018        let generation = self
6019            .registry
6020            .generation()
6021            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6022        let supervised = self.supervisor.list();
6023        let mut statuses = BTreeMap::new();
6024        let mut not_running = BTreeSet::new();
6025        for module in &supervised {
6026            let status = module.status_for_control("list").map_err(|err| {
6027                RouterError::backend(
6028                    0,
6029                    frame.header.corr,
6030                    format!("failed to read supervisor status: {err}"),
6031                )
6032            })?;
6033            if !status.enabled || !status.process_alive {
6034                not_running.insert(module.module_id().to_owned());
6035            }
6036            statuses.insert(module.module_id().to_owned(), status);
6037        }
6038        // Preserve cheap per-process readings before optional tree walks or
6039        // image hashing can spend the shared diagnostics deadline.
6040        let mut process_resources = BTreeMap::new();
6041        for module in &supervised {
6042            process_resources.insert(
6043                module.module_id().to_owned(),
6044                crate::subtree_resources::read_process(module.clone()).await,
6045            );
6046        }
6047        // Check image evidence before a process-table cache miss can spend the
6048        // remaining budget; a completed background hash must stay observable.
6049        let mut image_readings = BTreeMap::new();
6050        for module in &supervised {
6051            image_readings.insert(
6052                module.module_id().to_owned(),
6053                module.running_image_agreement().await,
6054            );
6055        }
6056        let readings = self
6057            .resource_tracker
6058            .read(supervised.clone(), false, "list")
6059            .await;
6060        let mut modules = Vec::new();
6061        for module in supervised {
6062            let status = statuses
6063                .remove(module.module_id())
6064                .expect("each supervised module has a status snapshot");
6065            let (configured, _) = module.configuration().map_err(|err| {
6066                RouterError::backend(
6067                    0,
6068                    frame.header.corr,
6069                    format!("failed to read module configuration: {err}"),
6070                )
6071            })?;
6072            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6073                RouterError::backend(
6074                    0,
6075                    frame.header.corr,
6076                    format!("failed to read retry time: {err}"),
6077                )
6078            })?;
6079            let image = image_readings
6080                .remove(module.module_id())
6081                .expect("each supervised module has an image snapshot");
6082            let resources = process_resources.remove(module.module_id());
6083            let mut verdict =
6084                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6085            verdict.spec = Some(launch_spec_agreement(
6086                &configured,
6087                status.process_alive,
6088                status.launch_spec_digest.as_ref(),
6089            ));
6090            let pending_reload = Some(verdict);
6091            let configured_enabled = self
6092                .capability_evaluator
6093                .configured_enabled(&status.module_id);
6094            // Keep the retired policy field on the wire for one release so
6095            // existing status consumers still receive the platform policy.
6096            modules.push(
6097                SupervisorEntry::new(
6098                    status.module_id,
6099                    status.state.to_string(),
6100                    status.enabled,
6101                    status.live,
6102                    status.health.status,
6103                )
6104                .with_configured_enabled(configured_enabled)
6105                .with_next_retry_at_ms(next_retry_at_ms)
6106                .with_launch_nonce_env(Some(!cfg!(unix)))
6107                .with_launch_nonce_source(status.launch_nonce_source)
6108                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6109                .with_protocol(status.protocol)
6110                .with_pending_reload(pending_reload)
6111                .with_last_probe_ms(status.health.last_probe_ms)
6112                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6113                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6114                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6115                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6116                .with_restart_count(Some(status.restart_count))
6117                .with_max_restarts(Some(status.max_restarts))
6118                .with_lifetime_restarts(Some(status.lifetime_restarts))
6119                .with_spawn_generation(Some(status.spawn_generation))
6120                .with_restart_window_secs(Some(status.restart_window.as_secs()))
6121                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6122                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6123                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6124                .with_resources(resources),
6125            );
6126        }
6127        for module in &mut modules {
6128            if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6129                module.tree = tree.clone();
6130                module.cpu_last_hour = hour.clone();
6131            }
6132            if module.tree.is_none() {
6133                module.tree_unavailable = Some(
6134                    if not_running.contains(&module.module_id) {
6135                        "not_running"
6136                    } else if readings.modules.contains_key(&module.module_id) {
6137                        "tree_unavailable"
6138                    } else {
6139                        "not_yet_measured"
6140                    }
6141                    .to_owned(),
6142                );
6143            }
6144        }
6145        let response = ClientControlResponse::SupervisorList {
6146            generation,
6147            modules,
6148            daemon: readings.daemon,
6149            agent_commands: readings.agent_commands,
6150        };
6151        Ok(vec![control_response_body_frame(
6152            &frame,
6153            &response,
6154            "ClientControlResponse::SupervisorList",
6155        )?])
6156    }
6157
6158    fn handle_supervisor_stderr_tail(
6159        &self,
6160        frame: Frame,
6161        module_id: String,
6162        max_lines: Option<u32>,
6163        max_bytes: Option<u32>,
6164    ) -> Result<Vec<Frame>, RouterError> {
6165        let Some(module) = self.supervisor.get(&module_id) else {
6166            return Ok(vec![control_error_frame(
6167                &frame,
6168                "unknown_module",
6169                format!("module_id '{module_id}' is not supervised"),
6170            )?]);
6171        };
6172
6173        let snapshot = module.stderr_tail(
6174            max_lines.map(|value| value as usize),
6175            max_bytes.map(|value| value as usize),
6176        );
6177
6178        let response = ClientControlResponse::SupervisorStderrTail {
6179            module_id,
6180            tail: StderrTail {
6181                capture: match snapshot.capture {
6182                    CaptureState::Captured => StderrCaptureState::Captured,
6183                    CaptureState::Incomplete { reason } => {
6184                        StderrCaptureState::Incomplete { reason }
6185                    }
6186                    CaptureState::NotCaptured { reason } => {
6187                        StderrCaptureState::NotCaptured { reason }
6188                    }
6189                },
6190                entries: snapshot
6191                    .entries
6192                    .into_iter()
6193                    .map(|entry| match entry {
6194                        TailEntry::Line {
6195                            text,
6196                            truncated,
6197                            at_ms,
6198                        } => StderrTailEntry::Line {
6199                            text,
6200                            truncated,
6201                            at_ms,
6202                        },
6203                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6204                    })
6205                    .collect(),
6206                dropped_lines: snapshot.dropped_lines,
6207            },
6208        };
6209        Ok(vec![control_response_body_frame(
6210            &frame,
6211            &response,
6212            "ClientControlResponse::SupervisorStderrTail",
6213        )?])
6214    }
6215
6216    async fn observe_control_wait<F: std::future::Future>(
6217        &self,
6218        ctx: &RouteCtx,
6219        target: Option<&str>,
6220        kind: &'static str,
6221        future: F,
6222    ) -> F::Output {
6223        self.counters
6224            .observe_reader_wait(
6225                || {
6226                    let module = self
6227                        .forwarding
6228                        .module_id_for_connection(ctx.connection_id)
6229                        .ok()
6230                        .flatten();
6231                    let principal = if module.is_some() { "module" } else { "direct" };
6232                    (
6233                        self.resolve_reader_wait_target(kind, target)
6234                            .or(module)
6235                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6236                        principal.into(),
6237                    )
6238                },
6239                kind,
6240                future,
6241            )
6242            .await
6243    }
6244
6245    /// Resolve the reader-wait module ID only from supervisor or registry
6246    /// records; never attribute a wait from the request selector alone.
6247    pub(crate) fn resolve_reader_wait_target(
6248        &self,
6249        op: &str,
6250        selector: Option<&str>,
6251    ) -> Option<String> {
6252        let selector = selector?;
6253        match op {
6254            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6255                .supervisor
6256                .get(selector)
6257                .map(|module| module.module_id().to_owned()),
6258            ops::SUPERVISOR_HEALTH_PROBE => self
6259                .registry
6260                .get_module(selector)
6261                .ok()
6262                .flatten()
6263                .map(|registration| registration.manifest.module_id),
6264            _ => None,
6265        }
6266    }
6267
6268    async fn handle_supervisor_terminals(
6269        &self,
6270        ctx: &RouteCtx,
6271        frame: Frame,
6272        module_id: String,
6273    ) -> Result<Vec<Frame>, RouterError> {
6274        let Some(module) = self.supervisor.get(&module_id) else {
6275            return Ok(vec![control_error_frame(
6276                &frame,
6277                "unknown_module",
6278                format!("module_id '{module_id}' is not supervised"),
6279            )?]);
6280        };
6281
6282        // The journal read runs on a blocking thread: it can be megabytes of
6283        // file I/O and must not occupy a runtime worker.
6284        let terminals = self
6285            .observe_control_wait(
6286                ctx,
6287                Some(module.module_id()),
6288                ops::SUPERVISOR_TERMINALS,
6289                module.read_durable_terminal_history(),
6290            )
6291            .await
6292            .map_err(|error| {
6293                RouterError::backend(
6294                    0,
6295                    frame.header.corr,
6296                    format!("failed to read terminal history: {error}"),
6297                )
6298            })?;
6299        let response = ClientControlResponse::SupervisorTerminals {
6300            module_id,
6301            terminals,
6302        };
6303        Ok(vec![control_response_body_frame(
6304            &frame,
6305            &response,
6306            "ClientControlResponse::SupervisorTerminals",
6307        )?])
6308    }
6309
6310    fn handle_supervisor_routes(
6311        &self,
6312        frame: Frame,
6313        module_id: Option<String>,
6314    ) -> Result<Vec<Frame>, RouterError> {
6315        let modules = self
6316            .forwarding
6317            .route_census(module_id.as_deref())
6318            .map_err(RouterError::Forwarding)?
6319            .into_iter()
6320            .map(|(module_id, routes)| SupervisorRouteModule {
6321                module_id,
6322                routes: routes
6323                    .into_iter()
6324                    .map(|route| SupervisorRoute {
6325                        consumer: match route.principal {
6326                            Principal::Reserved { module_id } => {
6327                                SupervisorRouteConsumer::Reserved { module_id }
6328                            }
6329                            Principal::Direct | Principal::Unverified => {
6330                                SupervisorRouteConsumer::Direct {
6331                                    connection_id: route.goodbye_target.connection_id.get(),
6332                                }
6333                            }
6334                        },
6335                        age_ms: Instant::now()
6336                            .saturating_duration_since(route.bound_at)
6337                            .as_millis()
6338                            .try_into()
6339                            .unwrap_or(u64::MAX),
6340                        draining: route.draining,
6341                        drain_reason: route.drain_reason,
6342                    })
6343                    .collect(),
6344            })
6345            .collect();
6346        let response = ClientControlResponse::SupervisorRoutes { modules };
6347        Ok(vec![control_response_body_frame(
6348            &frame,
6349            &response,
6350            "ClientControlResponse::SupervisorRoutes",
6351        )?])
6352    }
6353
6354    async fn handle_supervisor_provenance(
6355        &self,
6356        ctx: &RouteCtx,
6357        frame: Frame,
6358        module_id: Option<String>,
6359    ) -> Result<Vec<Frame>, RouterError> {
6360        let targeted = module_id.is_some();
6361        let selected = if let Some(module_id) = module_id {
6362            let Some(module) = self.supervisor.get(&module_id) else {
6363                return Ok(vec![control_error_frame(
6364                    &frame,
6365                    "unknown_module",
6366                    format!("module_id '{module_id}' is not supervised"),
6367                )?]);
6368            };
6369            vec![module]
6370        } else {
6371            self.supervisor.list()
6372        };
6373
6374        let target = targeted.then(|| selected[0].module_id());
6375        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6376            let mut modules = Vec::with_capacity(selected.len());
6377            for module in &selected {
6378                let (status, observed_image) = module
6379                    .status_and_running_image_agreement()
6380                    .await
6381                    .map_err(|err| {
6382                        RouterError::backend(
6383                            0,
6384                            frame.header.corr,
6385                            format!("failed to read supervisor status: {err}"),
6386                        )
6387                    })?;
6388                let module_declared = self
6389                    .registry
6390                    .get_module(&status.module_id)
6391                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6392                    .and_then(|registration| registration.manifest.provenance)
6393                    .map(|build| ModuleDeclaredProvenance::Reported { build })
6394                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6395                #[cfg(test)]
6396                let running_image = match &self.provenance_probe_override {
6397                    Some(result) => result.clone(),
6398                    None => observed_image,
6399                };
6400                #[cfg(not(test))]
6401                let running_image = observed_image;
6402                modules.push(SupervisorModuleProvenance {
6403                    module_id: status.module_id,
6404                    module_declared,
6405                    daemon_observed: SupervisorObservedProcess {
6406                        launch_nonce_source: status.launch_nonce_source,
6407                        pid: status.pid,
6408                        spawned_at_ms: status.spawned_at_ms,
6409                        spawned_from: status.spawned_from,
6410                        running_image,
6411                    },
6412                });
6413            }
6414            let daemon = SupervisorDaemonProvenance {
6415                daemon_build: self.daemon_provenance.build.clone(),
6416                daemon_observed: DaemonObservedProcess {
6417                    pid: self.daemon_provenance.pid,
6418                    started_at_ms: self
6419                        .daemon_provenance
6420                        .start_clock
6421                        .map(|clock| clock.started_at_ms())
6422                        .or(self.daemon_provenance.started_at_ms),
6423                    running_image: self
6424                        .daemon_provenance
6425                        .probe
6426                        .observe(
6427                            self.daemon_provenance.pid,
6428                            self.daemon_provenance.executable_path.as_deref(),
6429                            self.daemon_provenance.executable_identity.clone(),
6430                            self.daemon_provenance.process_start_time,
6431                        )
6432                        .await,
6433                },
6434            };
6435            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6436            Ok(vec![control_response_body_frame(
6437                &frame,
6438                &response,
6439                "ClientControlResponse::SupervisorProvenance",
6440            )?])
6441        })
6442        .await
6443    }
6444
6445    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6446        self.refresh_capability_requirements();
6447        let generation = self
6448            .registry
6449            .generation()
6450            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6451        let modules = self
6452            .supervisor
6453            .list()
6454            .into_iter()
6455            .map(|module| {
6456                let status = module.status_for_control("health").map_err(|err| {
6457                    RouterError::backend(
6458                        0,
6459                        frame.header.corr,
6460                        format!("failed to read supervisor health: {err}"),
6461                    )
6462                })?;
6463                let module_id = status.module_id;
6464                let capability_detail = self
6465                    .capability_evaluator
6466                    .required_problem_detail(&module_id);
6467                Ok(SupervisorHealthEntry {
6468                    module_id,
6469                    status: status.health.status,
6470                    detail: append_capability_problem_detail(
6471                        status.health.detail,
6472                        capability_detail,
6473                    ),
6474                    metrics: status.health.metrics,
6475                    consecutive_failures: status.health.consecutive_failures,
6476                    late_answer_count: status.health.late_answer_count,
6477                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6478                    last_action: status.health.last_action,
6479                    last_action_ms: status.health.last_action_ms,
6480                    last_probe_ms: status.health.last_probe_ms,
6481                })
6482            })
6483            .collect::<Result<Vec<_>, RouterError>>()?;
6484        let response = ClientControlResponse::SupervisorHealth {
6485            generation,
6486            modules,
6487        };
6488        Ok(vec![control_response_body_frame(
6489            &frame,
6490            &response,
6491            "ClientControlResponse::SupervisorHealth",
6492        )?])
6493    }
6494
6495    async fn handle_supervisor_restart(
6496        &self,
6497        frame: Frame,
6498        module_id: String,
6499        drain_timeout_ms: Option<u64>,
6500        requester: Principal,
6501    ) -> Result<Vec<Frame>, RouterError> {
6502        let operation_lock = self.supervisor.operation_lock();
6503        let _operation_guard = operation_lock.lock().await;
6504        let Some(module) = self.supervisor.get(&module_id) else {
6505            return Ok(vec![control_error_frame(
6506                &frame,
6507                "unknown_module",
6508                format!("module_id '{module_id}' is not supervised"),
6509            )?]);
6510        };
6511
6512        self.route_outages.mark_operator_action(&module_id);
6513        if let Err(err) = module
6514            .restart_requested(drain_timeout_ms, Some(requester))
6515            .await
6516        {
6517            self.route_outages
6518                .operator_action_ended_unrefused(&module_id);
6519            let (code, message) = match err {
6520                crate::supervise::SuperviseError::Disabled { .. } => {
6521                    ("module_disabled", err.to_string())
6522                }
6523                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6524                    ("swap_in_progress", err.to_string())
6525                }
6526                _ => (
6527                    "target_unavailable",
6528                    format!("failed to restart module_id '{module_id}': {err}"),
6529                ),
6530            };
6531            return Ok(vec![control_error_frame(&frame, code, message)?]);
6532        }
6533
6534        let response = ClientControlResponse::SupervisorAck {
6535            module_id,
6536            applied: true,
6537        };
6538        Ok(vec![control_response_body_frame(
6539            &frame,
6540            &response,
6541            "ClientControlResponse::SupervisorAck",
6542        )?])
6543    }
6544
6545    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
6546    /// when the old process has finished draining: a caller whose own lane
6547    /// rides the old process must get its reply before that drain waits on it.
6548    async fn handle_supervisor_swap(
6549        &self,
6550        frame: Frame,
6551        module_id: String,
6552        ready_timeout_ms: Option<u64>,
6553        requester: Principal,
6554    ) -> Result<Vec<Frame>, RouterError> {
6555        // The daemon-wide operation lock is held only to resolve the handle,
6556        // not across the swap. The swap can take its whole readiness budget,
6557        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6558        // holding it here would park an operator's stop behind the swap it is
6559        // meant to abort. A rescan or stop that reaches the module during the
6560        // swap is served by the swap itself (see `supervise_swap`).
6561        let module = {
6562            let operation_lock = self.supervisor.operation_lock();
6563            let _operation_guard = operation_lock.lock().await;
6564            self.supervisor.get(&module_id)
6565        };
6566        let Some(module) = module else {
6567            return Ok(vec![control_error_frame(
6568                &frame,
6569                "unknown_module",
6570                format!("module_id '{module_id}' is not supervised"),
6571            )?]);
6572        };
6573
6574        self.route_outages.mark_operator_action(&module_id);
6575        if let Err(err) = module
6576            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6577            .await
6578        {
6579            self.route_outages
6580                .operator_action_ended_unrefused(&module_id);
6581            use crate::supervise::SuperviseError;
6582            let message = err.to_string();
6583            let error = match err {
6584                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6585                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6586                    code: "swap_refused".to_string(),
6587                    message,
6588                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6589                },
6590                SuperviseError::SwapFailed {
6591                    arm,
6592                    candidate_exit,
6593                    ..
6594                } => ErrorBody {
6595                    code: "swap_failed".to_string(),
6596                    message,
6597                    detail: Some(serde_json::json!({
6598                        "arm": arm.as_str(),
6599                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6600                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6601                    })),
6602                },
6603                _ => ErrorBody::new(
6604                    "target_unavailable",
6605                    format!("failed to swap module_id '{module_id}': {message}"),
6606                ),
6607            };
6608            return Ok(vec![control_error_body_frame(&frame, error)?]);
6609        }
6610        // A completed swap kept the incumbent serving until cutover, so it
6611        // usually opened no outage; a mark left behind would make the next,
6612        // unrelated outage read as requested.
6613        self.route_outages
6614            .operator_action_ended_unrefused(&module_id);
6615
6616        let response = ClientControlResponse::SupervisorAck {
6617            module_id,
6618            applied: true,
6619        };
6620        Ok(vec![control_response_body_frame(
6621            &frame,
6622            &response,
6623            "ClientControlResponse::SupervisorAck",
6624        )?])
6625    }
6626
6627    async fn handle_supervisor_reload(
6628        &self,
6629        frame: Frame,
6630        module_id: String,
6631        requester: Principal,
6632    ) -> Result<Vec<Frame>, RouterError> {
6633        let operation_lock = self.supervisor.operation_lock();
6634        let _operation_guard = operation_lock.lock().await;
6635        let Some(module) = self.supervisor.get(&module_id) else {
6636            return Ok(vec![control_error_frame(
6637                &frame,
6638                "unknown_module",
6639                format!("module_id '{module_id}' is not supervised"),
6640            )?]);
6641        };
6642
6643        self.route_outages.mark_operator_action(&module_id);
6644        if let Err(err) = module.reload_requested(Some(requester)).await {
6645            self.route_outages
6646                .operator_action_ended_unrefused(&module_id);
6647            let (code, message) = match err {
6648                crate::supervise::SuperviseError::Disabled { .. } => {
6649                    ("module_disabled", err.to_string())
6650                }
6651                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6652                    ("swap_in_progress", err.to_string())
6653                }
6654                _ => (
6655                    "reload_failed",
6656                    format!("failed to reload module_id '{module_id}': {err}"),
6657                ),
6658            };
6659            return Ok(vec![control_error_frame(&frame, code, message)?]);
6660        }
6661
6662        let response = ClientControlResponse::SupervisorAck {
6663            module_id,
6664            applied: true,
6665        };
6666        Ok(vec![control_response_body_frame(
6667            &frame,
6668            &response,
6669            "ClientControlResponse::SupervisorAck",
6670        )?])
6671    }
6672
6673    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6674    /// the configuration comes from and whether to apply it; every mode then
6675    /// runs the same comparison, so a preview describes the apply it previews.
6676    async fn handle_supervisor_rescan(
6677        &self,
6678        frame: Frame,
6679        mode: RescanMode,
6680        requester: Principal,
6681    ) -> Result<Vec<Frame>, RouterError> {
6682        let preview = mode.previews();
6683        let Some(context) = self.rescan.clone() else {
6684            return Ok(vec![control_error_frame(
6685                &frame,
6686                "rescan_unavailable",
6687                "the daemon was not started with a reloadable config path".to_string(),
6688            )?]);
6689        };
6690
6691        let operation_lock = self.supervisor.operation_lock();
6692        let _operation_guard = operation_lock.lock().await;
6693        // Loaded under the operation lock, so no other rescan can change the
6694        // staged configuration between a checked apply's comparison and the
6695        // apply itself.
6696        let loaded = match self.load_rescan_config(&mode, &context) {
6697            Ok(config) => config,
6698            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6699        };
6700        // `load` reports a missing file as Ok(None), which is correct at boot
6701        // (no config, nothing to supervise) and catastrophic here: rescan treats
6702        // "not in the config" as "remove it", so an absent file would read as an
6703        // empty module list and retire the entire running fleet. An editor
6704        // writing via write-new-then-rename, or a half-finished edit, is enough
6705        // to open that window. Refuse instead: a config that cannot be read
6706        // carries no instruction to remove anything.
6707        let Some((config, activity_config)) = loaded else {
6708            return Ok(vec![control_error_frame(
6709                &frame,
6710                "invalid_daemon_config",
6711                format!(
6712                    "daemon config not found at {}; refusing to rescan (an absent config would \
6713                     retire every supervised module)",
6714                    context.config_path.display()
6715                ),
6716            )?]);
6717        };
6718        let (
6719            configured_port,
6720            storage_config,
6721            admission_facts_carrier_module_id,
6722            admission_facts_targets,
6723            scope_authority_owners,
6724            federated_scope_owner,
6725            privileged_operator_prompt,
6726            modules,
6727            reserved_capabilities,
6728        ) = (
6729            config.port,
6730            config.storage,
6731            config.admission_facts_carrier_module_id,
6732            config.admission_facts_targets,
6733            config.scope_authority_owners,
6734            config.federated_scope_owner,
6735            config.privileged_operator_prompt,
6736            config.modules,
6737            config.reserved_capabilities,
6738        );
6739
6740        // Collect the sections rescan cannot apply, so the REPLY carries them.
6741        //
6742        // The warning below has always been correct and has always gone only to
6743        // the journal -- addressed to whoever reads logs, while the person who
6744        // just edited the config is looking at the CLI. Naming each section
6745        // individually rather than setting a flag: "something outside modules
6746        // changed" sends the operator back to diffing their own file, which is
6747        // the work this is meant to save.
6748        let mut restart_required = Vec::new();
6749        for section in RestartRequiredSection::ALL {
6750            let changed = match section {
6751                RestartRequiredSection::Port => configured_port != context.configured_port,
6752                RestartRequiredSection::Storage => storage_config != context.storage_config,
6753                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6754                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6755                }
6756                RestartRequiredSection::AdmissionFactsTargets => {
6757                    admission_facts_targets != context.admission_facts_targets
6758                }
6759                RestartRequiredSection::ScopeAuthorityOwners => {
6760                    scope_authority_owners != context.scope_authority_owners
6761                }
6762                RestartRequiredSection::FederatedScopeOwner => {
6763                    federated_scope_owner != context.federated_scope_owner
6764                }
6765                RestartRequiredSection::PrivilegedOperatorPrompt => {
6766                    privileged_operator_prompt != context.privileged_operator_prompt
6767                }
6768            };
6769            if changed {
6770                restart_required.push(section.label().to_string());
6771            }
6772        }
6773        if !restart_required.is_empty() {
6774            warn!(
6775                config_path = %context.config_path.display(),
6776                sections = %restart_required.join(", "),
6777                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6778            );
6779        }
6780
6781        for configured in &modules {
6782            if let Err(err) = validate_spec(&configured.module_spec()) {
6783                return Ok(vec![control_error_frame(
6784                    &frame,
6785                    "invalid_daemon_config",
6786                    format!("supervisor rescan rejected daemon config: {err}"),
6787                )?]);
6788            }
6789        }
6790
6791        let configured_capabilities = modules
6792            .iter()
6793            .map(|module| (module.module_id.clone(), module.enabled))
6794            .collect::<Vec<_>>();
6795        let preview_capability_warnings = if preview {
6796            let (_, registrations) = self.runtime_capability_snapshot()?;
6797            let current_modules = self
6798                .supervisor
6799                .list()
6800                .into_iter()
6801                .map(|module| module.module_id().to_string())
6802                .collect::<BTreeSet<_>>();
6803            let resulting_modules = configured_capabilities.clone();
6804            let removed = current_modules
6805                .into_iter()
6806                .filter(|module_id| {
6807                    !resulting_modules
6808                        .iter()
6809                        .any(|(configured_id, _)| configured_id == module_id)
6810                })
6811                .collect::<Vec<_>>();
6812            // A candidate preview also names providers the candidate disables.
6813            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6814            let disabled = match &mode {
6815                RescanMode::Candidate(_) => resulting_modules
6816                    .iter()
6817                    .filter(|(_, enabled)| !enabled)
6818                    .map(|(module_id, _)| module_id.clone())
6819                    .collect::<Vec<_>>(),
6820                _ => Vec::new(),
6821            };
6822            self.capability_evaluator.preview_warnings(
6823                resulting_modules,
6824                &removed,
6825                &disabled,
6826                &registrations,
6827            )
6828        } else {
6829            Vec::new()
6830        };
6831        let reconciled = match &mode {
6832            RescanMode::Candidate(candidate) => self
6833                .preview_candidate(&context, modules, candidate)
6834                .map(|result| (result, Vec::new())),
6835            _ => {
6836                self.reconcile_supervised_modules(
6837                    &context.supervisor,
6838                    modules,
6839                    preview,
6840                    requester.clone(),
6841                )
6842                .await
6843            }
6844        };
6845        let (result, changed) = match reconciled {
6846            Ok(result) => result,
6847            Err(message) => {
6848                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6849            }
6850        };
6851        if !preview {
6852            if let Some(ledger) = &self.activity {
6853                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6854                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6855            }
6856            if let Err(code) = self.configure_activity(activity_config).await {
6857                warn!(
6858                    code,
6859                    "activity retention could not be applied during rescan"
6860                );
6861            }
6862            self.capability_evaluator
6863                .configure(configured_capabilities, reserved_capabilities);
6864            self.capability_evaluator.wake_deadline_loop();
6865            self.refresh_capability_requirements();
6866        }
6867        let mut result = result;
6868        result.restart_required = restart_required;
6869        result.capability_warnings = preview_capability_warnings;
6870        let response = ClientControlResponse::SupervisorRescan { result };
6871        Ok(vec![control_response_body_frame(
6872            &frame,
6873            &response,
6874            "ClientControlResponse::SupervisorRescan",
6875        )?])
6876    }
6877
6878    async fn handle_supervisor_release_reserved(
6879        &self,
6880        frame: Frame,
6881        module_id: String,
6882    ) -> Result<Vec<Frame>, RouterError> {
6883        let Some(context) = self.rescan.clone() else {
6884            return Ok(vec![control_error_frame(
6885                &frame,
6886                "release_unavailable",
6887                "reserved-id release requires a daemon started with a reloadable config path",
6888            )?]);
6889        };
6890        let operation_lock = self.supervisor.operation_lock();
6891        let _operation_guard = operation_lock.lock().await;
6892        let loaded = match crate::daemon_config::load(&context.config_path) {
6893            Ok(Some(config)) => config,
6894            Ok(None) => {
6895                return Ok(vec![control_error_frame(
6896                    &frame,
6897                    "invalid_daemon_config",
6898                    format!(
6899                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6900                        context.config_path.display()
6901                    ),
6902                )?])
6903            }
6904            Err(err) => {
6905                return Ok(vec![control_error_frame(
6906                    &frame,
6907                    "invalid_daemon_config",
6908                    format!("unable to verify reserved-id release against daemon config: {err}"),
6909                )?])
6910            }
6911        };
6912        if loaded
6913            .modules
6914            .iter()
6915            .any(|configured| configured.module_id == module_id)
6916        {
6917            return Ok(vec![control_error_frame(
6918                &frame,
6919                "reserved_module_configured",
6920                format!(
6921                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6922                ),
6923            )?]);
6924        }
6925        if !self.supervisor.release_retained_reserved_gate(&module_id) {
6926            return Ok(vec![control_error_frame(
6927                &frame,
6928                "reserved_gate_not_retained",
6929                format!(
6930                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6931                ),
6932            )?]);
6933        }
6934
6935        let response = ClientControlResponse::SupervisorAck {
6936            module_id,
6937            applied: true,
6938        };
6939        Ok(vec![control_response_body_frame(
6940            &frame,
6941            &response,
6942            "ClientControlResponse::SupervisorAck",
6943        )?])
6944    }
6945
6946    /// Compare the configured module set with the supervised one, changing
6947    /// nothing. The staged spec each supervised module holds is what the
6948    /// configured entry is compared with.
6949    fn plan_rescan(
6950        &self,
6951        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6952    ) -> Result<RescanPlan, String> {
6953        let mut current = BTreeMap::new();
6954        for module in self.supervisor.list() {
6955            let (spec, health) = module.configuration().map_err(|err| {
6956                format!(
6957                    "failed to read configuration for module_id '{}': {err}",
6958                    module.module_id()
6959                )
6960            })?;
6961            let enabled = module
6962                .status()
6963                .map_err(|err| {
6964                    format!(
6965                        "failed to read status for module_id '{}': {err}",
6966                        module.module_id()
6967                    )
6968                })?
6969                .enabled;
6970            current.insert(
6971                module.module_id().to_string(),
6972                (module, spec, health, enabled),
6973            );
6974        }
6975        let configured = configured_modules
6976            .into_iter()
6977            .map(|module| (module.module_id.clone(), module))
6978            .collect::<BTreeMap<_, _>>();
6979
6980        let added = configured
6981            .keys()
6982            .filter(|module_id| !current.contains_key(*module_id))
6983            .cloned()
6984            .collect::<Vec<_>>();
6985        let removed = current
6986            .keys()
6987            .filter(|module_id| !configured.contains_key(*module_id))
6988            .cloned()
6989            .collect::<Vec<_>>();
6990        let mut changed_pending_reload = Vec::new();
6991        let mut configuration_changes = BTreeSet::new();
6992        let mut enabled_changes = BTreeSet::new();
6993        let mut unchanged = 0_u32;
6994
6995        for (module_id, configured_module) in &configured {
6996            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6997            else {
6998                continue;
6999            };
7000            // Compare the whole launch spec so a future launch field cannot
7001            // accidentally become a live-only policy change. Health is stored
7002            // separately and applies live without replacing the process.
7003            let launch_changed = *current_spec != configured_module.module_spec();
7004            let configuration_changed =
7005                launch_changed || *current_health != configured_module.health;
7006            // Applying a rescan also replaces the recorded config value, even
7007            // when an operator already put the runtime in the requested state.
7008            let enabled_changed = *current_enabled != configured_module.enabled
7009                || self
7010                    .capability_evaluator
7011                    .configured_enabled(module_id)
7012                    .is_some_and(|enabled| enabled != configured_module.enabled);
7013            if configuration_changed {
7014                configuration_changes.insert(module_id.clone());
7015            }
7016            if launch_changed {
7017                changed_pending_reload.push(module_id.clone());
7018            }
7019            if enabled_changed {
7020                enabled_changes.insert(module_id.clone());
7021            }
7022            if !configuration_changed && !enabled_changed {
7023                unchanged = unchanged.saturating_add(1);
7024            }
7025        }
7026        Ok(RescanPlan {
7027            current,
7028            configured,
7029            added,
7030            removed,
7031            changed_pending_reload,
7032            configuration_changes,
7033            enabled_changes,
7034            unchanged,
7035        })
7036    }
7037
7038    /// Reconcile the running module set against the configured one.
7039    ///
7040    /// With `preview` set, the diff is computed and returned WITHOUT applying any
7041    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
7042    /// deliberately shares this function with the executing path rather than
7043    /// computing the same diff somewhere else -- two implementations of one
7044    /// decision agree until they do not, and the whole value of a preview is that
7045    /// it describes the operation that will actually run.
7046    async fn reconcile_supervised_modules(
7047        &self,
7048        supervisor: &Supervisor,
7049        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7050        preview: bool,
7051        requester: Principal,
7052    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7053        let plan = self.plan_rescan(configured_modules)?;
7054
7055        // Everything above this point is pure computation over two snapshots.
7056        // Everything below MUTATES. The preview returns here so the boundary is a
7057        // single early return rather than a condition repeated at each mutation
7058        // site, where one missed guard would apply part of a change the caller was
7059        // told would not happen.
7060        if preview {
7061            return Ok((plan.preview_result(), Vec::new()));
7062        }
7063        let changed = plan
7064            .configuration_changes
7065            .union(&plan.enabled_changes)
7066            .cloned()
7067            .collect::<Vec<_>>();
7068        let RescanPlan {
7069            current,
7070            configured,
7071            added,
7072            removed,
7073            changed_pending_reload,
7074            configuration_changes,
7075            enabled_changes,
7076            unchanged,
7077        } = plan;
7078        // The staged generation moves before the first change rather than
7079        // after the last, so a rescan that fails partway still invalidates
7080        // every earlier `supervisor.rescan_checked` preview: part of its diff
7081        // may have run.
7082        if !(added.is_empty()
7083            && removed.is_empty()
7084            && configuration_changes.is_empty()
7085            && enabled_changes.is_empty())
7086        {
7087            if let Some(context) = &self.rescan {
7088                context
7089                    .generation
7090                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7091            }
7092        }
7093
7094        for module_id in &removed {
7095            let module = &current
7096                .get(module_id)
7097                .expect("removed module came from current supervisor state")
7098                .0;
7099            module
7100                .retire_requested(Some(requester.clone()))
7101                .await
7102                .map_err(|err| {
7103                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
7104                })?;
7105            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
7106            //
7107            // `handle_route_open` resolves an absent module in three steps:
7108            // registry, then supervisor status, then tombstone. Retiring first
7109            // opens a window where ALL THREE ARE ABSENT -- the registry entry
7110            // went with the teardown above, the supervisor entry went with
7111            // `retire`, and the tombstone does not exist yet -- so a route.open
7112            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
7113            // it") for a module that was deliberately removed and whose caller
7114            // should get `module_removed` (TERMINAL, carrying a removal age).
7115            //
7116            // Writing the tombstone first closes it: during the window the
7117            // supervisor entry still answers, so the caller gets
7118            // `target_unavailable` -- retryable, and TRUE, because the module
7119            // is mid-teardown. After both statements it is `module_removed`.
7120            // No instant remains where a removed module reads as one that
7121            // never existed.
7122            //
7123            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
7124            // the absence of a test beside a fix invites deletion: these are
7125            // two sync statements with no await between them, so reaching the
7126            // window needs a second worker thread to land exactly between them
7127            // and there is no hook to force it. MEASURED: the 25 daemon_config
7128            // tests pass identically with the old order and the new one, so
7129            // the existing suite cannot see this and a green run is not
7130            // evidence either way. What the suite does hold is the
7131            // post-condition -- a removed module answers `module_removed` --
7132            // which this preserves.
7133            //
7134            // Found by an Athena panel reading the shipped tree against a
7135            // design note (2026-09-19), as the one concrete instance of that
7136            // note's class that survived contact with source. Direction is
7137            // benign: retryable where terminal was intended, never the reverse.
7138            self.supervisor.record_rescan_removal(module_id);
7139            self.supervisor.retire(module_id);
7140            self.route_outages.forget(module_id);
7141        }
7142
7143        for module_id in configured.keys() {
7144            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7145                continue;
7146            };
7147            let configured_module = configured
7148                .get(module_id)
7149                .expect("configured module id came from configured map");
7150            if configuration_changes.contains(module_id) {
7151                module
7152                    .update_configuration(
7153                        configured_module.module_spec(),
7154                        configured_module.health.clone(),
7155                        configured_module.drain_timeout_ms,
7156                    )
7157                    .await
7158                    .map_err(|err| {
7159                        format!(
7160                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
7161                        )
7162                    })?;
7163            }
7164            // Config-only drift is applied by the evaluator's config update;
7165            // do not turn it into a runtime start or stop.
7166            if *current_enabled != configured_module.enabled {
7167                // A rescan that starts or stops a module applies an operator's
7168                // edit to the config, so the resulting outage was asked for.
7169                self.route_outages.mark_operator_action(module_id);
7170                module
7171                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7172                    .await
7173                    .map_err(|err| {
7174                        self.route_outages.operator_action_ended_unrefused(module_id);
7175                        format!(
7176                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7177                            configured_module.enabled
7178                        )
7179                    })?;
7180            }
7181        }
7182
7183        for module_id in &added {
7184            let configured_module = configured
7185                .get(module_id)
7186                .expect("added module id came from configured map");
7187            supervisor
7188                .supervise_configured_requested(
7189                    configured_module.module_spec(),
7190                    configured_module.enabled,
7191                    configured_module.health.clone(),
7192                    configured_module.drain_timeout_ms,
7193                    configured_module.restart,
7194                    configured_module.register_timeout,
7195                    Some(requester.clone()),
7196                )
7197                .map_err(|err| {
7198                    format!("failed to add module_id '{module_id}' during rescan: {err}")
7199                })?;
7200        }
7201
7202        Ok((
7203            SupervisorRescanResult {
7204                added,
7205                removed,
7206                changed_pending_reload,
7207                enabled_changes: enabled_changes.iter().cloned().collect(),
7208                unchanged,
7209                preview: false,
7210                // Filled by the caller, which is the only layer that can see the
7211                // previous config to diff against.
7212                restart_required: Vec::new(),
7213                capability_warnings: Vec::new(),
7214                module_actions: Vec::new(),
7215                earlier_pending: Vec::new(),
7216                expect: None,
7217            },
7218            changed,
7219        ))
7220    }
7221
7222    async fn handle_supervisor_set_enabled(
7223        &self,
7224        frame: Frame,
7225        module_id: String,
7226        enabled: bool,
7227        requester: Principal,
7228    ) -> Result<Vec<Frame>, RouterError> {
7229        let operation_lock = self.supervisor.operation_lock();
7230        let _operation_guard = operation_lock.lock().await;
7231        let Some(module) = self.supervisor.get(&module_id) else {
7232            return Ok(vec![control_error_frame(
7233                &frame,
7234                "unknown_module",
7235                format!("module_id '{module_id}' is not supervised"),
7236            )?]);
7237        };
7238
7239        // Enabling counts as well as disabling: a module an operator starts
7240        // is refused until it registers, and that wait was asked for.
7241        self.route_outages.mark_operator_action(&module_id);
7242        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7243            Ok(applied) => applied,
7244            Err(err) => {
7245                self.route_outages
7246                    .operator_action_ended_unrefused(&module_id);
7247                return Ok(vec![control_error_frame(
7248                    &frame,
7249                    "target_unavailable",
7250                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7251                )?]);
7252            }
7253        };
7254        if !applied {
7255            // Already in the requested state: nothing was made unavailable,
7256            // so the mark must not outlive this request.
7257            self.route_outages
7258                .operator_action_ended_unrefused(&module_id);
7259        }
7260
7261        self.capability_evaluator.wake_deadline_loop();
7262        self.refresh_capability_requirements();
7263        let response = ClientControlResponse::SupervisorAck { module_id, applied };
7264        Ok(vec![control_response_body_frame(
7265            &frame,
7266            &response,
7267            "ClientControlResponse::SupervisorAck",
7268        )?])
7269    }
7270
7271    async fn handle_supervisor_health_probe(
7272        &self,
7273        ctx: &RouteCtx,
7274        frame: Frame,
7275        module_id: String,
7276    ) -> Result<Vec<Frame>, RouterError> {
7277        self.refresh_capability_requirements();
7278        let Some(registration) = self
7279            .registry
7280            .get_module(&module_id)
7281            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7282        else {
7283            return Ok(vec![control_error_frame(
7284                &frame,
7285                "unknown_module",
7286                format!("module_id '{module_id}' is not registered"),
7287            )?]);
7288        };
7289
7290        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
7291        // named for it. Making `module_registration_grants_op` return false
7292        // unconditionally reddens five tests, and every one is named for something
7293        // else -- capability relay, probe/bind demultiplexing, supervision-only
7294        // probing. They exercise a successful advertisement check on the way to their
7295        // own subject.
7296        //
7297        // Real protection, fragile in a specific way: narrowing any of those tests to
7298        // focus on its stated subject would silently remove coverage nobody knows
7299        // they are carrying. Recorded here rather than as a sixth test, because the
7300        // useful fact is WHICH tests hold the guard up -- a new test would add
7301        // coverage without telling the next person what the existing ones quietly do.
7302        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7303        {
7304            return Ok(vec![control_error_frame(
7305                &frame,
7306                "health_not_advertised",
7307                format!("module_id '{module_id}' did not advertise health.check"),
7308            )?]);
7309        }
7310
7311        self.observe_control_wait(
7312            ctx,
7313            Some(&registration.manifest.module_id),
7314            ops::SUPERVISOR_HEALTH_PROBE,
7315            async {
7316                let deadline = Instant::now() + self.health_probe_timeout;
7317                let pending = match self.forwarding.begin_module_control_rpc_for(
7318                    &module_id,
7319                    MODULE_CONTROL_OP_HEALTH_CHECK,
7320                    deadline,
7321                ) {
7322                    Ok(pending) => pending,
7323                    Err(err) => {
7324                        return Ok(vec![control_error_frame(
7325                            &frame,
7326                            forwarding_error_code(&err),
7327                            err.to_string(),
7328                        )?])
7329                    }
7330                };
7331
7332                let PendingModuleControlRpc {
7333                    endpoint,
7334                    module_sink,
7335                    negotiated_ver,
7336                    corr: probe_corr,
7337                    receiver,
7338                } = pending;
7339                let mut guard =
7340                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7341                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7342                    .map_err(|err| {
7343                        RouterError::backend(
7344                            0,
7345                            frame.header.corr,
7346                            format!("failed to encode health.check request: {err}"),
7347                        )
7348                    })?;
7349                let probe_frame = Frame::build_with_version(
7350                    negotiated_ver,
7351                    FrameType::Request,
7352                    control_flags(),
7353                    0,
7354                    0,
7355                    probe_corr,
7356                    probe_body,
7357                )
7358                .map_err(RouterError::FrameBuild)?;
7359
7360                // Queue admission and the reply share one deadline, so a module that
7361                // stops reading cannot park the requesting connection's reader forever.
7362                match timeout_at(deadline, module_sink.send(probe_frame)).await {
7363                    Ok(Ok(())) => {}
7364                    Ok(Err(err)) => {
7365                        return Ok(vec![control_error_frame(
7366                            &frame,
7367                            "target_unavailable",
7368                            err.to_string(),
7369                        )?]);
7370                    }
7371                    Err(_) => {
7372                        return Ok(vec![control_error_frame(
7373                            &frame,
7374                            "module_timeout",
7375                            format!(
7376                                "module_id '{module_id}' did not answer health.check within {:?}",
7377                                self.health_probe_timeout
7378                            ),
7379                        )?]);
7380                    }
7381                }
7382
7383                match timeout_at(deadline, receiver).await {
7384                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7385                        guard.disarm();
7386                        let Some(report) = response.health_report() else {
7387                            return Ok(vec![control_error_frame(
7388                                &frame,
7389                                "invalid_control_body",
7390                                "health.check RPC returned a non-health response",
7391                            )?]);
7392                        };
7393                        // Metrics go out whole here. The supervisor's cached snapshot
7394                        // caps this blob (see truncate_health_metrics), and this path
7395                        // exists precisely to answer without that cap -- so applying it
7396                        // here would leave no way to see what the cached view drops.
7397                        let HealthReport {
7398                            status,
7399                            detail,
7400                            metrics,
7401                        } = report;
7402                        let capability_detail = self
7403                            .capability_evaluator
7404                            .required_problem_detail(&module_id);
7405                        let response = ClientControlResponse::SupervisorHealthProbe {
7406                            module_id,
7407                            status,
7408                            detail: append_capability_problem_detail(detail, capability_detail),
7409                            metrics,
7410                        };
7411                        Ok(vec![control_response_body_frame(
7412                            &frame,
7413                            &response,
7414                            "ClientControlResponse::SupervisorHealthProbe",
7415                        )?])
7416                    }
7417                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7418                        guard.disarm();
7419                        // The module refused the health probe with its own body, so
7420                        // the refusal is relayed without the daemon-origin flag
7421                        // (bit 6): a client must not read it as the daemon's own
7422                        // refusal, which it may treat as never forwarded.
7423                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7424                    }
7425                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7426                        guard.disarm();
7427                        Ok(vec![control_error_frame(
7428                            &frame,
7429                            "target_unavailable",
7430                            message,
7431                        )?])
7432                    }
7433                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7434                        guard.disarm();
7435                        Ok(vec![control_error_frame(
7436                            &frame,
7437                            "invalid_control_body",
7438                            message,
7439                        )?])
7440                    }
7441                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7442                        guard.disarm();
7443                        Ok(vec![control_error_frame(
7444                            &frame,
7445                            "invalid_control_body",
7446                            format!("expected module-control op '{expected}', got '{actual}'"),
7447                        )?])
7448                    }
7449                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7450                        guard.disarm();
7451                        Ok(vec![control_error_frame(
7452                            &frame,
7453                            "module_timeout",
7454                            format!(
7455                                "module_id '{module_id}' answered health.check after {:?}",
7456                                self.health_probe_timeout
7457                            ),
7458                        )?])
7459                    }
7460                    Ok(Err(_)) => Ok(vec![control_error_frame(
7461                        &frame,
7462                        "target_unavailable",
7463                        "health.check waiter was canceled before the module responded",
7464                    )?]),
7465                    Err(_) => Ok(vec![control_error_frame(
7466                        &frame,
7467                        "module_timeout",
7468                        format!(
7469                            "module_id '{module_id}' did not answer health.check within {:?}",
7470                            self.health_probe_timeout
7471                        ),
7472                    )?]),
7473                }
7474            },
7475        )
7476        .await
7477    }
7478
7479    fn supervisor_status(
7480        &self,
7481        module_id: &str,
7482        corr: u64,
7483    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7484        self.supervisor
7485            .get(module_id)
7486            .map(|module| {
7487                let warming = module.is_warming_for_control("status").map_err(|err| {
7488                    RouterError::backend(
7489                        0,
7490                        corr,
7491                        format!(
7492                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
7493                        ),
7494                    )
7495                })?;
7496                module.status_for_control("status").map_err(|err| {
7497                    RouterError::backend(
7498                        0,
7499                        corr,
7500                        format!(
7501                            "failed to read supervisor status for module_id '{module_id}': {err}"
7502                        ),
7503                    )
7504                }).map(|status| (status, warming))
7505            })
7506            .transpose()
7507    }
7508
7509    fn guard_module_control_op(
7510        &self,
7511        frame: &Frame,
7512        module_id: &str,
7513        op: &str,
7514    ) -> Result<Option<Frame>, RouterError> {
7515        if self.module_grants_op(module_id, op, frame.header.corr)? {
7516            return Ok(None);
7517        }
7518
7519        Ok(Some(control_error_frame(
7520            frame,
7521            "op_not_allowed",
7522            format!("module_id '{module_id}' did not grant control op '{op}'"),
7523        )?))
7524    }
7525
7526    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7527        let Some(registration) = self
7528            .registry
7529            .get_module(module_id)
7530            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7531        else {
7532            return Ok(false);
7533        };
7534        Ok(module_registration_grants_op(&registration.control_ops, op))
7535    }
7536
7537    fn handle_status_update(
7538        &self,
7539        endpoint: ModuleEndpointId,
7540        frame: Frame,
7541    ) -> Result<Vec<Frame>, RouterError> {
7542        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7543            Ok(update) => update,
7544            Err(err) => {
7545                // Forward-compat: a newer module may push a channel-0 op this subc
7546                // version doesn't know. The control contract says unknown push ops
7547                // are IGNORED, never answered with an error. Only a malformed body
7548                // for an op we DO know is a real error worth surfacing.
7549                if is_known_module_push_op(&frame.body) {
7550                    return Ok(vec![control_error_frame(
7551                        &frame,
7552                        "invalid_control_body",
7553                        format!("malformed module control push body: {err}"),
7554                    )?]);
7555                }
7556                return Ok(Vec::new());
7557            }
7558        };
7559
7560        match update {
7561            ModuleControlPush::RouteStatus {
7562                route_channel,
7563                route_epoch,
7564                status,
7565            } => {
7566                self.forwarding
7567                    .cache_status(endpoint, route_channel, route_epoch, status)
7568                    .map_err(RouterError::Forwarding)?;
7569            }
7570        }
7571        Ok(Vec::new())
7572    }
7573
7574    fn handle_route_poll(
7575        &self,
7576        ctx: &RouteCtx,
7577        frame: Frame,
7578        route_channel: u16,
7579        route_epoch: u32,
7580        kind: PollKind,
7581    ) -> Result<Vec<Frame>, RouterError> {
7582        let snapshot = self
7583            .forwarding
7584            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7585            .map_err(RouterError::Forwarding)?;
7586        let response = match (kind, snapshot) {
7587            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7588                ClientControlResponse::RoutePoll {
7589                    route_channel,
7590                    route_epoch,
7591                    status,
7592                    live: None,
7593                }
7594            }
7595            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7596                route_channel,
7597                route_epoch,
7598                status: None,
7599                live: None,
7600            },
7601            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7602                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7603                // is what makes reporting `true` correct rather than a
7604                // confident guess. `process_live` returns None only when the
7605                // module id has no supervisor snapshot at all -- an
7606                // externally-started module the daemon did not spawn -- and
7607                // for those the supervisor has no opinion to offer, ever. It
7608                // is never None for a supervised module in an unknown state:
7609                // a supervised module always has a snapshot, and the answer
7610                // comes from `state == Running && process_alive`.
7611                //
7612                // The route is Bound, so the module completed a HELLO on a
7613                // live connection; "the process this route points at is
7614                // running" is therefore attested by the binding rather than
7615                // assumed. Reporting `false` for an unsupervised module would
7616                // be the actual lie -- it would tell a client its healthy
7617                // route is dead because the daemon does not manage the
7618                // process.
7619                //
7620                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7621                // module whose liveness is genuinely unknown, e.g. a snapshot
7622                // that has not been populated yet -- THIS DEFAULT BECOMES
7623                // WRONG and must split: unsupervised stays true, unknown
7624                // becomes null so the client can tell the two apart. The
7625                // response field is already `Option<bool>`, so the wire can
7626                // carry that distinction today.
7627                let live = self
7628                    .process_liveness
7629                    .as_ref()
7630                    .and_then(|source| source.process_live(&module_id))
7631                    .unwrap_or(true);
7632                ClientControlResponse::RoutePoll {
7633                    route_channel,
7634                    route_epoch,
7635                    status: None,
7636                    live: Some(live),
7637                }
7638            }
7639            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7640                route_channel,
7641                route_epoch,
7642                status: None,
7643                live: Some(false),
7644            },
7645        };
7646
7647        Ok(vec![control_response_body_frame(
7648            &frame,
7649            &response,
7650            "ClientControlResponse::RoutePoll",
7651        )?])
7652    }
7653
7654    pub(crate) fn observe_module_control_completion(
7655        &self,
7656        completion: ModuleControlRpcCompletion,
7657    ) -> bool {
7658        match completion {
7659            ModuleControlRpcCompletion::Unknown => false,
7660            ModuleControlRpcCompletion::Settled => true,
7661            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7662                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7663                info!(
7664                    module_id = %module_id,
7665                    latency_ms,
7666                    "late health.check answer proves the module is alive"
7667                );
7668                match self
7669                    .supervisor
7670                    .record_late_health_answer(&module_id, latency_ms)
7671                {
7672                    Ok(true) => {}
7673                    Ok(false) => debug!(
7674                        module_id = %module_id,
7675                        latency_ms,
7676                        "late health.check answer has no active supervisor snapshot"
7677                    ),
7678                    Err(err) => warn!(
7679                        module_id = %module_id,
7680                        latency_ms,
7681                        error = %err,
7682                        "failed to record late health.check answer"
7683                    ),
7684                }
7685                true
7686            }
7687        }
7688    }
7689
7690    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7691    /// the module connection whose frame is being handled, or to the client that
7692    /// relay was opened for.
7693    ///
7694    /// This runs on the MODULE connection's frame handler, where returning `Err`
7695    /// ends that connection -- and a module connection carries every client's
7696    /// routes to that module, so ending it costs the whole fleet its tools.
7697    /// `ConnectionClosing` carries the id of the connection that is closing, and
7698    /// when that id is a CLIENT's, the condition is entirely about that one
7699    /// client's route.open. A client-scoped condition has no authority over a
7700    /// shared module connection, so it is logged and the single relay is dropped:
7701    /// the client is going away, and `complete_pending_relay` already removed the
7702    /// relay before failing, so there is nothing left to settle. Anything that
7703    /// relay still reserved is released by that client's own connection teardown,
7704    /// which is already under way -- that is what "closing" means.
7705    ///
7706    /// Every other failure is a statement about THIS connection and stays fatal:
7707    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7708    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7709    /// frames correctly.
7710    fn refuse_to_end_module_connection_for_a_client(
7711        &self,
7712        module_connection_id: ConnectionId,
7713        corr: u64,
7714        err: ForwardingError,
7715    ) -> Result<(), RouterError> {
7716        if let ForwardingError::ConnectionClosing { connection_id } = err {
7717            if connection_id != module_connection_id {
7718                warn!(
7719                    module_connection_id = module_connection_id.get(),
7720                    client_connection_id = connection_id.get(),
7721                    corr,
7722                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7723                );
7724                return Ok(());
7725            }
7726        }
7727        Err(RouterError::Forwarding(err))
7728    }
7729
7730    fn handle_module_relay_response(
7731        &self,
7732        connection_id: ConnectionId,
7733        frame: Frame,
7734    ) -> Result<Vec<Frame>, RouterError> {
7735        let mut secondary_error = None;
7736        let outcome = match frame.header.ty {
7737            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7738                Ok(probe) if probe.op == "route.bind" => {
7739                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7740                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7741                            RouteBindRelayOutcome::Accepted
7742                        }
7743                        Ok(other) => {
7744                            let message =
7745                                format!("route.bind response carried unexpected body: {other:?}");
7746                            secondary_error = Some(control_error_frame(
7747                                &frame,
7748                                "invalid_control_body",
7749                                message.clone(),
7750                            )?);
7751                            RouteBindRelayOutcome::ModuleGone(message)
7752                        }
7753                        Err(err) => {
7754                            let message = format!("malformed route.bind response body: {err}");
7755                            secondary_error = Some(control_error_frame(
7756                                &frame,
7757                                "invalid_control_body",
7758                                message.clone(),
7759                            )?);
7760                            RouteBindRelayOutcome::ModuleGone(message)
7761                        }
7762                    }
7763                }
7764                Ok(probe) => {
7765                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7766                    {
7767                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7768                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7769                            "malformed {} response body: {err}",
7770                            probe.op
7771                        )),
7772                    };
7773                    let completion = self
7774                        .forwarding
7775                        .complete_module_control_rpc(
7776                            connection_id,
7777                            frame.header.corr,
7778                            Some(&probe.op),
7779                            outcome,
7780                        )
7781                        .map_err(RouterError::Forwarding)?;
7782                    if !self.observe_module_control_completion(completion) {
7783                        debug!(
7784                            connection_id = connection_id.get(),
7785                            corr = frame.header.corr,
7786                            op = %probe.op,
7787                            "dropping late or unknown module-control RPC response"
7788                        );
7789                    }
7790                    return Ok(Vec::new());
7791                }
7792                Err(err) => {
7793                    if let Some(expected_op) = self
7794                        .forwarding
7795                        .pending_module_control_op(connection_id, frame.header.corr)
7796                        .map_err(RouterError::Forwarding)?
7797                    {
7798                        let completion = self
7799                            .forwarding
7800                            .complete_module_control_rpc(
7801                                connection_id,
7802                                frame.header.corr,
7803                                None,
7804                                ModuleControlRpcOutcome::MalformedResponse(format!(
7805                                    "malformed {expected_op} response body: {err}"
7806                                )),
7807                            )
7808                            .map_err(RouterError::Forwarding)?;
7809                        if !self.observe_module_control_completion(completion) {
7810                            debug!(
7811                                connection_id = connection_id.get(),
7812                                corr = frame.header.corr,
7813                                "dropping late malformed module-control RPC response"
7814                            );
7815                        }
7816                        return Ok(Vec::new());
7817                    }
7818                    let message = format!("malformed route.bind response body: {err}");
7819                    secondary_error = Some(control_error_frame(
7820                        &frame,
7821                        "invalid_control_body",
7822                        message.clone(),
7823                    )?);
7824                    RouteBindRelayOutcome::ModuleGone(message)
7825                }
7826            },
7827            FrameType::Error => {
7828                if self
7829                    .forwarding
7830                    .pending_module_control_op(connection_id, frame.header.corr)
7831                    .map_err(RouterError::Forwarding)?
7832                    .is_some()
7833                {
7834                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7835                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7836                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7837                            "malformed module-control ERROR body: {err}"
7838                        )),
7839                    };
7840                    let completion = self
7841                        .forwarding
7842                        .complete_module_control_rpc(
7843                            connection_id,
7844                            frame.header.corr,
7845                            None,
7846                            outcome,
7847                        )
7848                        .map_err(RouterError::Forwarding)?;
7849                    if !self.observe_module_control_completion(completion) {
7850                        debug!(
7851                            connection_id = connection_id.get(),
7852                            corr = frame.header.corr,
7853                            "dropping late or unknown module-control RPC error"
7854                        );
7855                    }
7856                    return Ok(Vec::new());
7857                }
7858                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7859                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7860                    Err(err) => {
7861                        let message = format!("malformed route.bind ERROR body: {err}");
7862                        secondary_error = Some(control_error_frame(
7863                            &frame,
7864                            "invalid_control_body",
7865                            message.clone(),
7866                        )?);
7867                        RouteBindRelayOutcome::ModuleGone(message)
7868                    }
7869                }
7870            }
7871            ty => {
7872                return Ok(vec![control_error_frame(
7873                    &frame,
7874                    "unsupported_control_frame",
7875                    format!("unsupported module channel-0 frame {ty:?}"),
7876                )?])
7877            }
7878        };
7879
7880        let settled =
7881            self.forwarding
7882                .complete_pending_relay(connection_id, frame.header.corr, outcome);
7883        let completion = match settled {
7884            Ok(completion) => completion,
7885            Err(err) => {
7886                self.refuse_to_end_module_connection_for_a_client(
7887                    connection_id,
7888                    frame.header.corr,
7889                    err,
7890                )?;
7891                return Ok(secondary_error.into_iter().collect());
7892            }
7893        };
7894        if let Some(target) = completion.abandoned.as_ref() {
7895            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7896        }
7897        if !completion.settled {
7898            debug!(
7899                connection_id = connection_id.get(),
7900                corr = frame.header.corr,
7901                frame_type = ?frame.header.ty,
7902                "dropping late or unknown route.bind relay response"
7903            );
7904        }
7905        Ok(secondary_error.into_iter().collect())
7906    }
7907
7908    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7909        debug!(connection_id = connection_id.get(), "handling GOODBYE");
7910        // GOODBYE ends the connection's logical session even when its socket
7911        // stays open. Use disconnect teardown so verdicts, client notices and
7912        // scope authority are released at the same lifecycle boundary.
7913        self.cleanup_connection_with_end_reason(
7914            connection_id,
7915            RegistrationEndReason::ExplicitGoodbye,
7916        )
7917        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7918        Ok(Vec::new())
7919    }
7920}
7921
7922impl Default for ControlHandler {
7923    fn default() -> Self {
7924        Self::new(Arc::new(Registry::default()))
7925    }
7926}
7927
7928impl crate::supervise::SwapPromotionObserver for ControlHandler {
7929    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7930        self.apply_registration_capabilities(registration);
7931    }
7932}
7933
7934fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7935    CapabilityRequirementStatus {
7936        consumer: status.consumer,
7937        capability: status.capability,
7938        need: match status.need {
7939            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7940            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7941        },
7942        verdict: status.verdict.as_str().to_string(),
7943        episode_seq: status.episode_seq,
7944        config_satisfiable: status.config_satisfiable,
7945        runtime_available: status.runtime_available,
7946        detail: status.detail,
7947    }
7948}
7949
7950fn append_capability_problem_detail(
7951    detail: Option<String>,
7952    capability_detail: Option<String>,
7953) -> Option<String> {
7954    match (detail, capability_detail) {
7955        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7956        (Some(detail), None) => Some(detail),
7957        (None, Some(capability_detail)) => Some(capability_detail),
7958        (None, None) => None,
7959    }
7960}
7961
7962fn subc_ops() -> Vec<String> {
7963    SUBC_CONTROL_OPS
7964        .iter()
7965        .map(|op| (*op).to_string())
7966        .collect()
7967}
7968
7969fn module_subc_ops() -> Vec<String> {
7970    SUBC_CONTROL_OPS
7971        .iter()
7972        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7973        .map(|op| (*op).to_string())
7974        .collect()
7975}
7976
7977#[cfg(test)]
7978fn module_baseline_control_ops() -> Vec<String> {
7979    MODULE_BASELINE_CONTROL_OPS
7980        .iter()
7981        .map(|op| (*op).to_string())
7982        .collect()
7983}
7984
7985fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7986    let mut seen = HashSet::new();
7987    let mut effective = Vec::new();
7988    for op in MODULE_BASELINE_CONTROL_OPS {
7989        if seen.insert((*op).to_string()) {
7990            effective.push((*op).to_string());
7991        }
7992    }
7993    for op in declared.unwrap_or_default() {
7994        if seen.insert(op.clone()) {
7995            effective.push(op);
7996        }
7997    }
7998    effective
7999}
8000
8001fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8002    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8003}
8004
8005fn target_module_id(target: &RouteTarget) -> &str {
8006    match target {
8007        RouteTarget::ToolProvider { module_id }
8008        | RouteTarget::ManagementSurface { module_id }
8009        | RouteTarget::InternalService { module_id, .. } => module_id,
8010    }
8011}
8012
8013fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8014    roles.iter().any(|role| match (target, role) {
8015        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8016        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8017        (
8018            RouteTarget::InternalService { service_id, .. },
8019            ProviderRole::InternalService {
8020                service_id: provided,
8021                ..
8022            },
8023        ) => service_id == provided,
8024        _ => false,
8025    })
8026}
8027
8028fn is_routable_role(role: &ProviderRole) -> bool {
8029    matches!(
8030        role,
8031        ProviderRole::ToolProvider { .. }
8032            | ProviderRole::ManagementSurface { .. }
8033            | ProviderRole::InternalService { .. }
8034    )
8035}
8036
8037#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8038enum ControlRequestBodyError {
8039    UnknownOp,
8040    InvalidBody,
8041}
8042
8043#[derive(Debug, Deserialize)]
8044struct ControlOpProbe {
8045    op: String,
8046}
8047
8048/// Channel-0 push ops this subc version understands. A push whose `op` is not in
8049/// this set is treated as a forward-compat unknown and ignored rather than errored.
8050const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8051
8052fn is_known_module_push_op(body: &[u8]) -> bool {
8053    serde_json::from_slice::<ControlOpProbe>(body)
8054        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8055        .unwrap_or(false)
8056}
8057
8058fn is_known_module_request_op(body: &[u8]) -> bool {
8059    serde_json::from_slice::<ControlOpProbe>(body)
8060        .map(|probe| is_module_to_subc_op(&probe.op))
8061        .unwrap_or(false)
8062}
8063
8064fn is_module_to_subc_op(op: &str) -> bool {
8065    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8066}
8067
8068fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8069    debug!(
8070        op = %op,
8071        connection_id = connection_id.get(),
8072        corr,
8073        "control dispatch"
8074    );
8075}
8076
8077fn log_slow_control_dispatch(
8078    dispatch_started_at: Option<StdInstant>,
8079    op: &'static str,
8080    connection_id: ConnectionId,
8081    corr: u64,
8082    route_open_timings: Option<&RouteOpenTimings>,
8083) {
8084    let Some(dispatch_started_at) = dispatch_started_at else {
8085        return;
8086    };
8087    // Use the same monotonic clock as the phase timers and relay deadline.
8088    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8089    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8090        if let Some(timings) = route_open_timings {
8091            warn!(
8092                op = %op,
8093                connection_id = connection_id.get(),
8094                corr,
8095                elapsed_ms = elapsed.as_millis() as u64,
8096                module_id = ?timings.module_id,
8097                principal = %timings.principal,
8098                admission_ms = timings.admission.as_millis() as u64,
8099                project_root_ms = timings.project_root.as_millis() as u64,
8100                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8101                bind_send_ms = timings.bind_send.as_millis() as u64,
8102                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8103                "slow control dispatch"
8104            );
8105            return;
8106        }
8107        warn!(
8108            op = %op,
8109            connection_id = connection_id.get(),
8110            corr,
8111            elapsed_ms = elapsed.as_millis() as u64,
8112            "slow control dispatch"
8113        );
8114    }
8115}
8116
8117/// A test stand-in for the OS caller lookup, given the recorded endpoints.
8118#[cfg(test)]
8119type TestCallerLookup =
8120    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8121
8122/// Every new client operation needs an explicit privilege decision here.
8123fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8124    match request {
8125        ClientControlRequest::SupervisorRestart { module_id, .. } => {
8126            Some(PrivilegedOperation::Restart { module_id })
8127        }
8128        ClientControlRequest::SupervisorSwap { module_id, .. } => {
8129            Some(PrivilegedOperation::Swap { module_id })
8130        }
8131        ClientControlRequest::SupervisorReload { module_id } => {
8132            Some(PrivilegedOperation::Reload { module_id })
8133        }
8134        ClientControlRequest::SupervisorRescan { preview } => {
8135            if *preview {
8136                None
8137            } else {
8138                Some(PrivilegedOperation::Rescan)
8139            }
8140        }
8141        // Only the well-formed apply asks for approval. A preview changes
8142        // nothing, and any other combination is refused before it touches
8143        // anything, so prompting the operator for it would ask about a request
8144        // that cannot run.
8145        ClientControlRequest::SupervisorRescanChecked {
8146            preview,
8147            candidate,
8148            expect,
8149        } => (!*preview && candidate.is_none() && expect.is_some())
8150            .then_some(PrivilegedOperation::Rescan),
8151        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8152            Some(PrivilegedOperation::SetEnabled {
8153                module_id,
8154                enabled: *enabled,
8155            })
8156        }
8157        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8158            Some(PrivilegedOperation::ReleaseReserved { module_id })
8159        }
8160        ClientControlRequest::ServerDescribe {}
8161        // The announcement uses the same client admission as supervisor.restart,
8162        // but cannot stop anything itself and needs no operator confirmation.
8163        | ClientControlRequest::DaemonShutdownIntent { .. }
8164        | ClientControlRequest::CatalogList { .. }
8165        | ClientControlRequest::RouteOpen { .. }
8166        | ClientControlRequest::RoutePoll { .. }
8167        | ClientControlRequest::SupervisorList {}
8168        | ClientControlRequest::SupervisorHealth {}
8169        | ClientControlRequest::SupervisorHealthProbe { .. }
8170        | ClientControlRequest::SupervisorRoutes { .. }
8171        | ClientControlRequest::SupervisorProvenance { .. }
8172        | ClientControlRequest::SupervisorStderrTail { .. }
8173        | ClientControlRequest::SupervisorTerminals { .. }
8174        | ClientControlRequest::SupervisorSpawnSnapshot {}
8175        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8176    }
8177}
8178
8179fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8180    match request {
8181        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8182        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8183        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8184        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8185        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8186        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8187        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8188        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8189        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8190        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8191        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8192        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8193        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8194        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8195        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8196        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8197        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8198        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8199        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8200        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8201        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8202    }
8203}
8204
8205/// The caller of one `route.open`, as its `route.open refused` line names it.
8206///
8207/// A connection id alone cannot name a caller after the fact: ids restart at
8208/// every daemon restart, so the same number belongs to different modules
8209/// before and after one. These three fields are what the refused line needs
8210/// to be attributable on its own, and they are rendered with the same helpers
8211/// as the `route.open accepted` line so both directions grep alike.
8212struct RouteOpenCaller {
8213    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
8214    /// when the request presented a consumer identity whose launch nonce did
8215    /// not verify. The claimed module id is never printed as a principal.
8216    principal: String,
8217    /// `none` for an unscoped open; the admitted stamp's label once scope
8218    /// admission issued one; before that, the requested selector's label.
8219    scope: String,
8220    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
8221    /// because it is a string the caller chose.
8222    harness: String,
8223}
8224
8225impl RouteOpenCaller {
8226    fn new(
8227        principal: Option<&Principal>,
8228        scope: Option<&ScopeSelector>,
8229        identity: &BindIdentity,
8230    ) -> Self {
8231        Self {
8232            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8233            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8234            harness: bounded_log_text(&identity.harness),
8235        }
8236    }
8237
8238    /// For a refusal that has no parsed request at hand: names whatever the
8239    /// frame still says, and `unknown` where it says nothing parseable.
8240    fn unknown() -> Self {
8241        Self {
8242            principal: "unknown".to_string(),
8243            scope: "unknown".to_string(),
8244            harness: "unknown".to_string(),
8245        }
8246    }
8247}
8248
8249/// Longest client-chosen text (a scope ref, a harness) a route.open log line
8250/// prints before cutting it.
8251const MAX_LOGGED_TEXT_CHARS: usize = 64;
8252
8253/// `value` Debug-quoted, so control characters land escaped and cannot forge
8254/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
8255/// `…` inside the quotes so one request cannot flood the line.
8256fn bounded_log_text(value: &str) -> String {
8257    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8258    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8259        bounded.push('…');
8260    }
8261    format!("{bounded:?}")
8262}
8263
8264/// The `scope=` field of the `route.open accepted` line, from the stamp that
8265/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
8266/// so a ref holding control characters cannot forge a log line, and cut to 64
8267/// characters with a trailing `…` to bound the line. The bind proof and the
8268/// scope's attributes never appear.
8269fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8270    use subc_protocol::scope::ScopeKind;
8271    let kind = match stamp.kind {
8272        ScopeKind::Head => "head",
8273        ScopeKind::Worker => "worker",
8274        ScopeKind::Ephemeral => "ephemeral",
8275        ScopeKind::Federated => "federated",
8276        _ => "unknown",
8277    };
8278    format!(
8279        "{kind}/{}/{}@{}",
8280        principal_label(&stamp.owner),
8281        bounded_log_text(&stamp.scope_ref),
8282        stamp.scope_epoch
8283    )
8284}
8285
8286/// The `scope=` field of a `route.open refused` line for an open refused
8287/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
8288/// The kind is not known yet, so the literal `requested` stands in its place.
8289/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
8290/// owner it has not been checked against the configured modules: a quoted
8291/// owner means the value was never validated. The bind proof never appears.
8292fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8293    let epoch = selector
8294        .scope_epoch
8295        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8296    format!(
8297        "requested/{}/{}@{epoch}",
8298        bounded_log_text(&principal_label(&selector.owner)),
8299        bounded_log_text(&selector.scope_ref),
8300    )
8301}
8302
8303fn principal_label(principal: &Principal) -> String {
8304    match principal {
8305        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8306        Principal::Direct => "direct".to_string(),
8307        other => format!("{other:?}"),
8308    }
8309}
8310
8311fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8312    match request {
8313        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8314        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8315        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8316        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8317        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8318        ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8319        _ => "unsupported",
8320    }
8321}
8322
8323fn parse_client_control_request(
8324    body: &[u8],
8325) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8326    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8327        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8328            Ok(probe)
8329                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8330                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8331            {
8332                ControlRequestBodyError::InvalidBody
8333            }
8334            Ok(_) => ControlRequestBodyError::UnknownOp,
8335            Err(_) => ControlRequestBodyError::InvalidBody,
8336        };
8337        (err, classification)
8338    })
8339}
8340
8341fn parse_module_control_request_from_module(
8342    body: &[u8],
8343) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8344    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8345        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8346            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8347            Ok(_) => ControlRequestBodyError::UnknownOp,
8348            Err(_) => ControlRequestBodyError::InvalidBody,
8349        };
8350        (err, classification)
8351    })
8352}
8353
8354#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8355enum ProviderRoleKind {
8356    ToolProvider,
8357    PipelineStage,
8358    ManagementSurface,
8359    InternalService,
8360}
8361
8362fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8363    match role {
8364        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8365        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8366        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8367        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8368    }
8369}
8370
8371fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8372    roles.iter().map(provider_role_kind).collect()
8373}
8374
8375/// Most refused scope records named individually in the log per sync; the
8376/// `refused` count on the accepted line is always complete.
8377const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8378
8379/// Per-outcome counts of one accepted `scope.sync`, for its log line.
8380#[derive(Debug, Default, PartialEq, Eq)]
8381struct ScopeOutcomeCounts {
8382    created: usize,
8383    replaced: usize,
8384    updated: usize,
8385    unchanged: usize,
8386    refused: usize,
8387}
8388
8389impl ScopeOutcomeCounts {
8390    fn of(results: &[ScopeRecordResult]) -> Self {
8391        let mut counts = Self::default();
8392        for result in results {
8393            let slot = match result.outcome {
8394                ScopeRecordOutcome::Created => &mut counts.created,
8395                ScopeRecordOutcome::Replaced => &mut counts.replaced,
8396                ScopeRecordOutcome::Updated => &mut counts.updated,
8397                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8398                ScopeRecordOutcome::Refused => &mut counts.refused,
8399            };
8400            *slot += 1;
8401        }
8402        counts
8403    }
8404}
8405
8406#[cfg(test)]
8407mod scope_outcome_count_tests {
8408    use super::*;
8409
8410    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8411        ScopeRecordResult {
8412            scope_ref: "r".to_string(),
8413            scope_epoch: 1,
8414            outcome,
8415            code: None,
8416            message: None,
8417            version: None,
8418            parent_state: None,
8419        }
8420    }
8421
8422    /// Each outcome lands in its own count, so a refused record can never be
8423    /// hidden inside the total the log already printed.
8424    #[test]
8425    fn every_outcome_is_counted_in_its_own_field() {
8426        let results = [
8427            result(ScopeRecordOutcome::Created),
8428            result(ScopeRecordOutcome::Created),
8429            result(ScopeRecordOutcome::Replaced),
8430            result(ScopeRecordOutcome::Updated),
8431            result(ScopeRecordOutcome::Unchanged),
8432            result(ScopeRecordOutcome::Refused),
8433            result(ScopeRecordOutcome::Refused),
8434            result(ScopeRecordOutcome::Refused),
8435        ];
8436        assert_eq!(
8437            ScopeOutcomeCounts::of(&results),
8438            ScopeOutcomeCounts {
8439                created: 2,
8440                replaced: 1,
8441                updated: 1,
8442                unchanged: 1,
8443                refused: 3,
8444            }
8445        );
8446    }
8447}
8448
8449/// Return whether a catalog change can create a newly violating live route.
8450/// Removing an attested claim is intentionally excluded: it makes fewer routes
8451/// forbidden and therefore must leave the existing route census untouched.
8452fn capability_census_trigger(
8453    old: Option<&CapabilityDeclarations>,
8454    new: Option<&CapabilityDeclarations>,
8455) -> bool {
8456    let old_provides = old
8457        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8458        .unwrap_or_default();
8459    let old_denies = old
8460        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8461        .unwrap_or_default();
8462    let new = new.cloned().unwrap_or(CapabilityDeclarations {
8463        provides: Vec::new(),
8464        requires: Vec::new(),
8465        must_never_reach: Vec::new(),
8466    });
8467
8468    new.provides
8469        .iter()
8470        .any(|capability| !old_provides.contains(capability))
8471        || new
8472            .must_never_reach
8473            .iter()
8474            .any(|capability| !old_denies.contains(capability))
8475}
8476
8477/// Find the first capability an attested opener denies that an attested target
8478/// claims. Both manifests are live registry records, never cached or client data.
8479fn denied_capability<'a>(
8480    opening_manifest: &'a ModuleManifest,
8481    target_manifest: &ModuleManifest,
8482) -> Option<&'a str> {
8483    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8484    let target_capabilities = target_manifest.capabilities.as_ref()?;
8485    opening_capabilities
8486        .must_never_reach
8487        .iter()
8488        .find(|denied| {
8489            target_capabilities
8490                .provides
8491                .iter()
8492                .any(|provided| provided == *denied)
8493        })
8494        .map(String::as_str)
8495}
8496
8497fn catalog_update_frozen_field_message(
8498    registered: &ModuleManifest,
8499    provides: &[ProviderRole],
8500) -> Option<String> {
8501    let old_has_provides = !registered.provides.is_empty();
8502    let new_has_provides = !provides.is_empty();
8503    if old_has_provides != new_has_provides {
8504        return Some(format!(
8505            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8506            registered.module_id
8507        ));
8508    }
8509
8510    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
8511        return Some(format!(
8512            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8513            registered.module_id
8514        ));
8515    }
8516
8517    let registered_concurrency = manifest_concurrency(registered);
8518    let mut candidate = registered.clone();
8519    candidate.provides = provides.to_vec();
8520    let candidate_concurrency = manifest_concurrency(&candidate);
8521    if candidate_concurrency != registered_concurrency {
8522        return Some(format!(
8523            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8524            registered.module_id, registered_concurrency, candidate_concurrency
8525        ));
8526    }
8527
8528    // control_ops live beside the manifest in the HELLO body, not inside
8529    // ModuleManifest, so a provides-only catalog.update cannot change them.
8530    None
8531}
8532
8533fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8534    manifest.provides.iter().any(is_routable_role)
8535}
8536
8537/// Returns the routable-provider concurrency subc should enforce for this manifest.
8538///
8539/// ToolProvider and ManagementSurface store their delivery concurrency directly.
8540/// InternalService has no role-specific concurrency field, so it retains the
8541/// existing ModuleManaged default for backward compatibility.
8542fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8543    manifest
8544        .provides
8545        .iter()
8546        .find_map(|provider| match provider {
8547            ProviderRole::ToolProvider { concurrency, .. }
8548            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8549            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8550        })
8551        .unwrap_or(Concurrency::ModuleManaged)
8552}
8553
8554/// True when the manifest carries a ManagementSurface role whose concurrency
8555/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
8556/// bytes because the typed manifest deliberately erases that distinction: the
8557/// default exists for wire compatibility, and this probe exists so the default
8558/// stays observable. Any parse irregularity returns false -- the caller only
8559/// logs, and a malformed body already failed registration upstream.
8560fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8561    let has_management_surface = manifest
8562        .provides
8563        .iter()
8564        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8565    if !has_management_surface {
8566        return false;
8567    }
8568    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8569        return false;
8570    };
8571    let Some(provides) = raw
8572        .get("manifest")
8573        .and_then(|manifest| manifest.get("provides"))
8574        .and_then(serde_json::Value::as_array)
8575    else {
8576        return false;
8577    };
8578    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
8579    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
8580    // against the management_surface_manifest_without_concurrency golden, not
8581    // recalled (the externally-tagged guess was this function's first bug).
8582    provides.iter().any(|role| {
8583        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8584            && role.get("concurrency").is_none()
8585    })
8586}
8587
8588fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8589    if peer_version != PROTOCOL_VERSION {
8590        return Err(format!(
8591            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8592        ));
8593    }
8594    Ok(PROTOCOL_VERSION)
8595}
8596
8597fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8598    Frame::build_with_version(
8599        response_version(frame),
8600        FrameType::Pong,
8601        frame.header.flags,
8602        0,
8603        0,
8604        frame.header.corr,
8605        Vec::new(),
8606    )
8607    .map_err(RouterError::FrameBuild)
8608}
8609
8610pub(crate) fn control_error_frame(
8611    frame: &Frame,
8612    code: &'static str,
8613    message: impl Into<String>,
8614) -> Result<Frame, RouterError> {
8615    control_error_body_frame(
8616        frame,
8617        ErrorBody {
8618            code: code.to_string(),
8619            message: message.into(),
8620            detail: None,
8621        },
8622    )
8623}
8624
8625/// A channel-0 ERROR for a refusal the daemon decided itself: every
8626/// route.open refusal, the drain `module_reloading` and scope-change/end
8627/// bodies, spawn-subscription and supervisor refusals. Carries the
8628/// daemon-origin bit, which is what lets a client treat `module_reloading`
8629/// as proof that its request never reached a module. Response, push,
8630/// goodbye and relay-request frames keep the plain control flags in their
8631/// own builders.
8632fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8633    control_error_frame_with_flags(frame, error, daemon_error_flags())
8634}
8635
8636/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8637/// refused route.bind answered as a route.open refusal, or a refused
8638/// module-control RPC. Plain control flags — the code and message are
8639/// module-chosen, so the frame must not claim daemon authorship, even when
8640/// the module picks a daemon code like `module_reloading`.
8641fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8642    control_error_frame_with_flags(frame, error, control_flags())
8643}
8644
8645fn control_error_frame_with_flags(
8646    frame: &Frame,
8647    error: ErrorBody,
8648    flags: Flags,
8649) -> Result<Frame, RouterError> {
8650    let body = serde_json::to_vec(&error).map_err(|err| {
8651        RouterError::backend(
8652            0,
8653            frame.header.corr,
8654            format!("failed to encode control ERROR: {err}"),
8655        )
8656    })?;
8657
8658    Frame::build_with_version(
8659        response_version(frame),
8660        FrameType::Error,
8661        flags,
8662        0,
8663        0,
8664        frame.header.corr,
8665        body,
8666    )
8667    .map_err(RouterError::FrameBuild)
8668}
8669
8670pub(crate) fn control_response_body_frame<T: Serialize>(
8671    frame: &Frame,
8672    reply: &T,
8673    label: &'static str,
8674) -> Result<Frame, RouterError> {
8675    let body = serde_json::to_vec(reply).map_err(|err| {
8676        RouterError::backend(
8677            0,
8678            frame.header.corr,
8679            format!("failed to encode {label}: {err}"),
8680        )
8681    })?;
8682
8683    Frame::build_with_version(
8684        response_version(frame),
8685        FrameType::Response,
8686        control_flags(),
8687        0,
8688        0,
8689        frame.header.corr,
8690        body,
8691    )
8692    .map_err(RouterError::FrameBuild)
8693}
8694
8695/// Map a forwarding failure to the wire code a client sees.
8696///
8697/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8698/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8699/// chosen here decides whether a caller retries or gives up.
8700///
8701/// That makes attribution the load-bearing property, not merely having a code. A
8702/// permanent fault published as a retryable one produces a fleet-wide retry storm
8703/// against something that can never recover; a transient fault published as
8704/// permanent gives up on work that would have succeeded. Both look correct in a
8705/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8706/// the mapping per variant rather than merely asserting that some code exists.
8707///
8708/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8709/// two codes on the same side of the boundary passes it. Measured rather than
8710/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8711/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8712/// a test named for something else that happens to assert the string.
8713///
8714/// That accidental coverage is deliberately left alone rather than promoted to a
8715/// named test, because it guards a property this function does not promise.
8716/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8717/// specific code within a class, so identity is free to change and only the
8718/// partition is a contract. Splitting it out would assert a guarantee nothing
8719/// depends on — and a suite that promises more than the code does is the harder
8720/// thing to correct later, because the next reader cannot tell which assertions
8721/// are load-bearing.
8722///
8723/// Pin identity here the moment a consumer branches on a specific code.
8724fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8725    match err {
8726        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8727        ForwardingError::NoModuleConnection => "target_unavailable",
8728        ForwardingError::ModuleReloading { .. } => "module_reloading",
8729        ForwardingError::ClientRouteChannelExhausted { .. }
8730        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8731        ForwardingError::StaleModuleEndpoint
8732        | ForwardingError::UnknownReservation { .. }
8733        | ForwardingError::ConnectionClosing { .. }
8734        | ForwardingError::ClientEgressClosed { .. }
8735        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8736        // Only a swap candidate's registration can produce this, and it means
8737        // exactly what a second active HELLO for a live id means.
8738        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8739        ForwardingError::RelayCorrelationExhausted
8740        | ForwardingError::RouteOpenBuild(_)
8741        | ForwardingError::Poisoned => "forwarding_error",
8742    }
8743}
8744
8745fn response_version(frame: &Frame) -> u8 {
8746    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8747        frame.header.ver
8748    } else {
8749        PROTOCOL_VERSION
8750    }
8751}
8752
8753fn control_flags() -> Flags {
8754    Flags::new(false, Priority::Passive, false)
8755}
8756
8757/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8758/// channel. The target is the module (a client never saw the route), so this
8759/// takes the module path: delivered late rather than dropped when the module's
8760/// queue is momentarily full, and never closing its connection.
8761fn send_goodbye_target_best_effort(
8762    counters: &DaemonCounters,
8763    target: &GoodbyeTarget,
8764    context: &'static str,
8765) {
8766    let Ok(frame) = Frame::build_with_version(
8767        target.negotiated_ver,
8768        FrameType::Goodbye,
8769        control_flags(),
8770        target.channel,
8771        target.epoch,
8772        0,
8773        Vec::new(),
8774    ) else {
8775        return;
8776    };
8777    crate::forwarding::send_module_route_goodbye(
8778        counters,
8779        &target.sink,
8780        frame,
8781        target.module_id.as_deref(),
8782        context,
8783    );
8784}
8785
8786pub(crate) fn send_route_control_pushes(
8787    forwarding: &ForwardingTable,
8788    routes: Vec<EndpointRoute>,
8789    push: ClientControlPush,
8790) {
8791    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8792    for route in routes {
8793        let target = route.goodbye_target;
8794        if let Some((existing, channels)) = targets
8795            .iter_mut()
8796            .find(|(existing, _)| existing.connection_id == target.connection_id)
8797        {
8798            debug_assert_eq!(
8799                existing.negotiated_ver, target.negotiated_ver,
8800                "one connection cannot negotiate multiple frame versions"
8801            );
8802            if !channels.contains(&target.channel) {
8803                channels.push(target.channel);
8804            }
8805            continue;
8806        }
8807        let channel = target.channel;
8808        targets.push((target, vec![channel]));
8809    }
8810    for (target, mut channels) in targets {
8811        channels.sort_unstable();
8812        let mut push = push.clone();
8813        match &mut push {
8814            ClientControlPush::RouteClosing {
8815                channels: covered, ..
8816            }
8817            | ClientControlPush::RouteClosed {
8818                channels: covered, ..
8819            } => *covered = channels,
8820        }
8821        let body = match serde_json::to_vec(&push) {
8822            Ok(body) => body,
8823            Err(err) => {
8824                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8825                continue;
8826            }
8827        };
8828        let frame = match Frame::build_with_version(
8829            target.negotiated_ver,
8830            FrameType::Push,
8831            control_flags(),
8832            0,
8833            0,
8834            0,
8835            body.clone(),
8836        ) {
8837            Ok(frame) => frame,
8838            Err(err) => {
8839                warn!(
8840                    route_channel = target.channel,
8841                    error = %err,
8842                    "failed to build route lifecycle control PUSH frame"
8843                );
8844                continue;
8845            }
8846        };
8847        if let Err(err) = target.sink.try_send(frame) {
8848            if target.close_on_delivery_failure() {
8849                warn!(
8850                    target_connection_id = target.connection_id.get(),
8851                    route_channel = target.channel,
8852                    error = %err,
8853                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8854                );
8855                let _ = forwarding.escalate_client_delivery_failure(
8856                    target.connection_id,
8857                    target.channel,
8858                    target.epoch,
8859                    CloseReason::new(
8860                        "route_lifecycle_push_delivery_failed",
8861                        format!(
8862                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8863                            target.channel
8864                        ),
8865                    ),
8866                    crate::forwarding::UndeliveredFrame {
8867                        module_id: target.module_id.as_deref(),
8868                        sink: &target.sink,
8869                    },
8870                );
8871            }
8872        }
8873    }
8874}
8875
8876#[cfg(test)]
8877mod tests {
8878    include!("control/agent_commands_tests.rs");
8879    include!("control/resource_revision_tests.rs");
8880    #[tokio::test]
8881    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8882        let registry = Arc::new(Registry::default());
8883        let handler = ControlHandler::new(registry);
8884        assert_eq!(
8885            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8886            None
8887        );
8888        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8889        hello_via_sink(
8890            &handler,
8891            &ctx,
8892            &mut rx,
8893            hello_frame("daemon", PROTOCOL_VERSION, 1),
8894        )
8895        .await;
8896        assert_eq!(
8897            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8898            Some("daemon".into())
8899        );
8900        assert_eq!(
8901            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8902            None
8903        );
8904        assert_eq!(
8905            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8906            None,
8907            "a registered module is not necessarily supervised"
8908        );
8909        assert_eq!(
8910            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8911            None,
8912            "aggregate provenance has no single target"
8913        );
8914        assert_eq!(
8915            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8916            None
8917        );
8918    }
8919
8920    mod peer_manifest {
8921        include!("peer_manifest_tests.rs");
8922    }
8923    mod activity_read {
8924        include!("activity_read_tests.rs");
8925    }
8926
8927    mod lifecycle_activity {
8928        include!("control_lifecycle_tests.rs");
8929    }
8930    #[cfg(unix)]
8931    #[tokio::test]
8932    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8933        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8934        let path = dir.join("subc.jsonc");
8935        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8936            "program":"/bin/sleep","args":["60"],"protocol":"none",
8937            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8938        }}}).to_string()).unwrap();
8939        let mut configured = crate::daemon_config::load(&path)
8940            .unwrap()
8941            .unwrap()
8942            .modules
8943            .pop()
8944            .unwrap();
8945        let registry = std::sync::Arc::new(crate::Registry::default());
8946        let handle = crate::SupervisorHandle::new();
8947        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8948            .with_handle(handle.clone());
8949        let module = supervisor
8950            .supervise_configured_with_health(
8951                configured.module_spec(),
8952                true,
8953                configured.health.clone(),
8954                None,
8955                configured.restart,
8956            )
8957            .unwrap();
8958        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8959        let before = module.status().unwrap().pid;
8960        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8961        configured.health.cadence = std::time::Duration::from_secs(3600);
8962        let health_only = handler
8963            .reconcile_supervised_modules(
8964                &supervisor,
8965                vec![configured.clone()],
8966                false,
8967                Principal::Direct,
8968            )
8969            .await
8970            .unwrap();
8971        assert!(
8972            health_only.0.changed_pending_reload.is_empty(),
8973            "health policy is already applied live"
8974        );
8975        assert_eq!(module.status().unwrap().pid, before);
8976        assert_eq!(
8977            module.configuration().unwrap().1.http,
8978            configured.health.http
8979        );
8980        configured.args = vec!["61".into()];
8981        let launch = handler
8982            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8983            .await
8984            .unwrap();
8985        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8986        assert_eq!(
8987            module.status().unwrap().pid,
8988            before,
8989            "a launch edit is stored until reload"
8990        );
8991        module.drain().await.unwrap();
8992    }
8993    use cortexkit_test_support::ScratchDir;
8994    use std::{
8995        collections::BTreeMap,
8996        fmt,
8997        path::PathBuf,
8998        sync::{Arc, Mutex},
8999        time::Duration,
9000    };
9001
9002    use serde_json::{json, Value};
9003    use subc_protocol::{
9004        manifest::{
9005            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9006            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9007        },
9008        session::HealthStatus,
9009        FrameType,
9010    };
9011
9012    use super::*;
9013    use crate::{
9014        forwarding::{DataRoute, DataRouteState},
9015        registry::ChannelState,
9016        router::FrameSink,
9017        stderr_tail::DEFAULT_MAX_LINE_BYTES,
9018        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9019        RouteCtx, Router,
9020    };
9021    use tokio::{
9022        sync::mpsc,
9023        time::{sleep, Instant},
9024    };
9025    use tracing::{
9026        field::{Field, Visit},
9027        Event, Subscriber,
9028    };
9029    use tracing_subscriber::{layer::Context, prelude::*, Layer};
9030
9031    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
9032    ///
9033    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
9034    /// is only populated for `tests/*.rs` integration test binaries -- this file
9035    /// compiles as part of the library target, which gets neither. This test's
9036    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
9037    /// and the sibling binary lives two directories up at
9038    /// `<target-dir>/<profile>/fake-aft-stub`.
9039    ///
9040    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
9041    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
9042    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
9043    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
9044    /// `NotFound`, which reads as a broken test rather than an unbuilt
9045    /// dependency -- so state the cause and the remedy instead. Deliberately a
9046    /// panic and not a silent skip: a test that quietly passes when it could not
9047    /// run is worse than one that fails, because it reports health it never
9048    /// verified.
9049    fn fake_aft_stub_path() -> PathBuf {
9050        let mut path = std::env::current_exe().expect("current_exe available in tests");
9051        path.pop(); // .../deps/
9052        path.pop(); // .../<profile>/
9053        path.push(if cfg!(windows) {
9054            "fake-aft-stub.exe"
9055        } else {
9056            "fake-aft-stub"
9057        });
9058        assert!(
9059            path.exists(),
9060            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9061             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9062            path.display()
9063        );
9064        path
9065    }
9066
9067    /// Whether clients retry `code` in place: the predicate itself, never a copy
9068    /// of its set. A copied list breaks silently when a code is added to or
9069    /// removed from the real one, and a stale copy here would let exactly the
9070    /// failure this test exists to catch pass.
9071    fn client_retries(code: &str) -> bool {
9072        subc_protocol::error_codes::is_retryable_route_open(code)
9073    }
9074
9075    /// A code is not a label — clients branch on it, so publishing the wrong KIND
9076    /// of failure is worse than publishing none. A permanent fault dressed as
9077    /// retryable makes every client in the fleet retry forever against something
9078    /// that cannot recover; a transient fault dressed as permanent abandons work
9079    /// that would have succeeded.
9080    ///
9081    /// Asserting "a code exists" cannot catch either, because the string is free
9082    /// to say anything. This enumerates every variant and pins which side of the
9083    /// retry boundary it lands on, so a new variant must be classified here
9084    /// deliberately rather than inheriting whichever arm it was appended to.
9085    #[test]
9086    fn retryability_of_forwarding_codes_matches_the_failure() {
9087        // Transient by nature: the target is booting, reloading, or its endpoint
9088        // was swapped mid-flight. Retrying is how these resolve.
9089        let transient = [
9090            ForwardingError::NoModuleConnection,
9091            ForwardingError::ModuleReloading {
9092                module_id: "m".into(),
9093            },
9094            ForwardingError::StaleModuleEndpoint,
9095            ForwardingError::UnknownReservation {
9096                client_channel: 1,
9097                module_channel: 1,
9098            },
9099            ForwardingError::ConnectionClosing {
9100                connection_id: ConnectionId::new(1),
9101            },
9102            ForwardingError::ClientEgressClosed {
9103                connection_id: ConnectionId::new(1),
9104            },
9105            ForwardingError::ModuleEgressUnavailable {
9106                connection_id: ConnectionId::new(1),
9107            },
9108        ];
9109        for err in transient {
9110            let code = forwarding_error_code(&err);
9111            assert!(
9112                client_retries(code),
9113                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9114            );
9115        }
9116
9117        // Not fixed by retrying. Channel and correlation exhaustion need the
9118        // caller to close routes, and a poisoned lock is a daemon that cannot
9119        // recover at all — the worst thing to advertise as retryable, since every
9120        // client would storm a daemon that will never answer.
9121        let permanent = [
9122            ForwardingError::ConnectionRoleConflict {
9123                connection_id: ConnectionId::new(1),
9124            },
9125            ForwardingError::ClientRouteChannelExhausted {
9126                connection_id: ConnectionId::new(1),
9127            },
9128            ForwardingError::ModuleRouteChannelExhausted {
9129                endpoint: ModuleEndpointId {
9130                    connection_id: ConnectionId::new(1),
9131                    generation: 1,
9132                },
9133            },
9134            ForwardingError::RelayCorrelationExhausted,
9135            ForwardingError::RouteOpenBuild("x".into()),
9136            ForwardingError::Poisoned,
9137        ];
9138        for err in permanent {
9139            let code = forwarding_error_code(&err);
9140            assert!(
9141                !client_retries(code),
9142                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9143            );
9144        }
9145    }
9146
9147    /// The principal is the daemon's answer to "who is calling", and modules
9148    /// branch on it: aft gates bash on it, cerebellum gates browser control,
9149    /// plexus gates connector invocation. So a stamp is an authorization input in
9150    /// another process, not a label — and both possible answers SUCCEED, which is
9151    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
9152    /// first-party capability to something that never proved it; a supervised one
9153    /// stamped `Direct` silently strips a module of capability it is entitled to.
9154    ///
9155    /// Neither shows up in a test that only checks the bind succeeded. Before this
9156    /// test the only coverage was accidental —
9157    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
9158    /// stamped principal on its way past, so narrowing that wire-shape test to its
9159    /// stated subject would have deleted the last assertion on this value. It
9160    /// still asserts the stamp, which is now redundancy rather than the only
9161    /// guard: both fail under the same mutation, and this one names the reason.
9162    /// SCOPE: this handler's supervisor has spawned nothing, so
9163    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
9164    /// is unreachable here. Both assertions below are refusals, and a mutant that
9165    /// refuses everything would satisfy them.
9166    ///
9167    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
9168    /// spawns a supervised consumer, reads its live nonce, and asserts the module
9169    /// observed `principal.kind == "reserved"` carrying that module_id — verified
9170    /// at source rather than assumed, since a citation is a claim about another
9171    /// file and ages like one. Recorded because a harness that structurally
9172    /// cannot reach an arm reports "none" for that arm identically to one that
9173    /// covers it and found nothing.
9174    #[tokio::test]
9175    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9176        let handler = ControlHandler::default();
9177        let frame =
9178            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9179
9180        // Absent consumer_identity is the ordinary case: a human at a terminal, or
9181        // any process holding the connection file. Nothing was proved, so nothing
9182        // may be granted beyond the unattested floor.
9183        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9184        assert_eq!(
9185            stamped,
9186            Principal::Direct,
9187            "a caller that proved nothing must not be stamped as a supervised module"
9188        );
9189
9190        // A claimed module_id with a nonce no supervised child was given is a
9191        // forgery attempt, not a weaker caller: it must be REFUSED rather than
9192        // quietly demoted to Direct, or an impersonation attempt looks identical
9193        // to an ordinary unattested connection.
9194        let forged = handler
9195            .route_open_principal(
9196                &frame,
9197                Some(ConsumerIdentity {
9198                    module_id: "aft".to_string(),
9199                    launch_nonce: "not-a-real-nonce".to_string(),
9200                }),
9201            )
9202            .unwrap();
9203        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9204        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9205    }
9206
9207    /// The test above hands `route_open_principal` an identity it built itself,
9208    /// which proves the stamping rule and nothing about where the identity comes
9209    /// from. The real producer is a wire body, and the two are joined by a serde
9210    /// field name that nothing else asserts.
9211    ///
9212    /// That join fails quietly in one specific way: an unrecognised key is simply
9213    /// absent after parsing, so a renamed or misspelled `consumer_identity`
9214    /// yields `None` and every supervised module silently drops to `Direct`.
9215    /// Capability-wise that is the safe direction, but it surfaces far from its
9216    /// cause — as a module mysteriously refused bash — and it would pass every
9217    /// test that builds its own input.
9218    ///
9219    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
9220    /// would break every client the moment the daemon gains a field, trading a
9221    /// quiet demotion for a hard refusal on additive change. Asserting the join
9222    /// instead means a rename breaks a test here rather than the fleet.
9223    #[test]
9224    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9225        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9226        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9227        let ClientControlRequest::RouteOpen {
9228            consumer_identity, ..
9229        } = parsed
9230        else {
9231            panic!("route.open body must parse as RouteOpen");
9232        };
9233        assert_eq!(
9234            consumer_identity,
9235            Some(ConsumerIdentity {
9236                module_id: "aft".to_string(),
9237                launch_nonce: "n".to_string(),
9238            }),
9239            "the wire field name must reach the value route_open_principal reads"
9240        );
9241    }
9242
9243    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9244        ModuleManifest::builder(module_id, "0.1.0")
9245            .protocol_ver(protocol_ver)
9246            .provides(vec![ProviderRole::ToolProvider {
9247                tools: vec![Tool {
9248                    name: "read".to_string(),
9249                    description: None,
9250                    execution_mode: ExecutionMode::Pure,
9251                    schema: json!({"type": "object"}),
9252                }],
9253                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9254                concurrency: Concurrency::ModuleManaged,
9255                emits_push: true,
9256                sub_supervises: true,
9257            }])
9258            .build()
9259    }
9260
9261    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9262        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9263    }
9264
9265    fn hello_frame_with_control_ops(
9266        module_id: &str,
9267        protocol_ver: u8,
9268        corr: u64,
9269        control_ops: Option<Vec<String>>,
9270    ) -> Frame {
9271        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9272    }
9273
9274    fn hello_frame_with_nonce(
9275        module_id: &str,
9276        protocol_ver: u8,
9277        corr: u64,
9278        launch_nonce: Option<&str>,
9279    ) -> Frame {
9280        hello_frame_full(
9281            module_id,
9282            protocol_ver,
9283            corr,
9284            None,
9285            launch_nonce.map(ToOwned::to_owned),
9286        )
9287    }
9288
9289    fn hello_frame_full(
9290        module_id: &str,
9291        protocol_ver: u8,
9292        corr: u64,
9293        control_ops: Option<Vec<String>>,
9294        launch_nonce: Option<String>,
9295    ) -> Frame {
9296        let body = serde_json::to_vec(&ModuleHelloBody {
9297            manifest: manifest(module_id, protocol_ver),
9298            protocol_ver,
9299            control_ops,
9300            launch_nonce,
9301        })
9302        .unwrap();
9303        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9304    }
9305
9306    fn hello_frame_with_rich_ui(
9307        module_id: &str,
9308        protocol_ver: u8,
9309        corr: u64,
9310        launch_nonce: Option<&str>,
9311        rich_ui: Value,
9312    ) -> Frame {
9313        let mut body = serde_json::to_value(ModuleHelloBody {
9314            manifest: manifest(module_id, protocol_ver),
9315            protocol_ver,
9316            control_ops: None,
9317            launch_nonce: launch_nonce.map(str::to_string),
9318        })
9319        .unwrap();
9320        body["manifest"]["rich_ui"] = rich_ui;
9321        Frame::build(
9322            FrameType::Hello,
9323            control_flags(),
9324            0,
9325            0,
9326            corr,
9327            serde_json::to_vec(&body).unwrap(),
9328        )
9329        .unwrap()
9330    }
9331
9332    fn non_routable_hello_frame_with_control_ops(
9333        module_id: &str,
9334        corr: u64,
9335        control_ops: Option<Vec<String>>,
9336    ) -> Frame {
9337        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9338        manifest.provides.clear();
9339        let body = serde_json::to_vec(&ModuleHelloBody {
9340            manifest,
9341            protocol_ver: PROTOCOL_VERSION,
9342            control_ops,
9343            launch_nonce: None,
9344        })
9345        .unwrap();
9346        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9347    }
9348
9349    fn capability_grammar_hello_frame(
9350        capabilities: Value,
9351        runtime_computed: Option<Value>,
9352        corr: u64,
9353    ) -> Frame {
9354        let mut body = serde_json::to_value(ModuleHelloBody {
9355            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9356            protocol_ver: PROTOCOL_VERSION,
9357            control_ops: None,
9358            launch_nonce: None,
9359        })
9360        .expect("HELLO body serializes");
9361        body["manifest"]["capabilities"] = capabilities;
9362        if let Some(runtime_computed) = runtime_computed {
9363            body["runtime_computed"] = runtime_computed;
9364        }
9365        Frame::build(
9366            FrameType::Hello,
9367            control_flags(),
9368            0,
9369            0,
9370            corr,
9371            serde_json::to_vec(&body).expect("HELLO body reserializes"),
9372        )
9373        .expect("HELLO frame builds")
9374    }
9375
9376    fn channel_request(channel: u16, corr: u64) -> Frame {
9377        Frame::build(
9378            FrameType::Request,
9379            Flags::new(true, Priority::Interactive, false),
9380            channel,
9381            0,
9382            corr,
9383            b"opaque".to_vec(),
9384        )
9385        .unwrap()
9386    }
9387
9388    fn route_ctx(
9389        connection_id: ConnectionId,
9390    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9391        let (tx, rx) = mpsc::channel(8);
9392        (
9393            RouteCtx {
9394                connection_id,
9395                egress: FrameSink::new(tx),
9396            },
9397            rx,
9398        )
9399    }
9400
9401    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9402        serde_json::from_slice(&frame.body).unwrap()
9403    }
9404
9405    /// Register a module over a connection that has a sink and return the
9406    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
9407    /// module's own sink rather than returning it as a reply, so the ack is
9408    /// taken off `rx` here and whatever the test reads next is what followed it.
9409    async fn hello_via_sink(
9410        handler: &ControlHandler,
9411        ctx: &RouteCtx,
9412        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9413        hello: Frame,
9414    ) -> Frame {
9415        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9416        assert!(
9417            replies.is_empty(),
9418            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9419        );
9420        let ack = rx
9421            .try_recv()
9422            .expect("HELLO_ACK is queued on the module sink")
9423            .frame;
9424        assert_eq!(ack.header.ty, FrameType::HelloAck);
9425        ack
9426    }
9427
9428    fn parse_error(frame: &Frame) -> Value {
9429        serde_json::from_slice(&frame.body).unwrap()
9430    }
9431
9432    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9433        serde_json::from_slice(&frame.body).unwrap()
9434    }
9435
9436    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9437        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9438            route_channel,
9439            route_epoch: 0,
9440            kind,
9441        })
9442        .unwrap();
9443        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9444    }
9445
9446    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9447        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9448            module_id: module_id.to_string(),
9449        })
9450        .unwrap();
9451        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9452    }
9453
9454    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9455        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9456    }
9457
9458    // Exercise retired opener fields without putting them back in BindIdentity.
9459    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9460        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9461        let mut frame = route_open_frame(corr, module_id, root);
9462        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9463        body["identity"]["project_root"] = serde_json::json!(path.as_path());
9464        frame.body = serde_json::to_vec(&body).unwrap();
9465        frame
9466    }
9467
9468    fn route_open_frame_with_consumer_capabilities(
9469        corr: u64,
9470        module_id: &str,
9471        _project_root: ScratchDir,
9472        consumer_capabilities: Option<Vec<String>>,
9473    ) -> Frame {
9474        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9475            busy_refusals: None,
9476            target: RouteTarget::ToolProvider {
9477                module_id: module_id.to_string(),
9478            },
9479            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9480            consumer_identity: None,
9481            consumer_capabilities,
9482            role_versions: None,
9483            admission_facts: None,
9484            scope: None,
9485        })
9486        .unwrap();
9487        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9488    }
9489
9490    fn route_open_frame_with_role_versions(
9491        corr: u64,
9492        module_id: &str,
9493        _project_root: ScratchDir,
9494        role_versions: Option<BTreeMap<String, String>>,
9495    ) -> Frame {
9496        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9497            busy_refusals: None,
9498            target: RouteTarget::ToolProvider {
9499                module_id: module_id.to_string(),
9500            },
9501            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9502            consumer_identity: None,
9503            consumer_capabilities: None,
9504            role_versions,
9505            admission_facts: None,
9506            scope: None,
9507        })
9508        .unwrap();
9509        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9510    }
9511
9512    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9513        entries
9514            .iter()
9515            .map(|(role, version)| (role.to_string(), version.to_string()))
9516            .collect()
9517    }
9518
9519    fn route_open_frame_with_admission_facts(
9520        corr: u64,
9521        module_id: &str,
9522        _project_root: ScratchDir,
9523        consumer_identity: Option<subc_control::ConsumerIdentity>,
9524        facts: Option<Value>,
9525    ) -> Frame {
9526        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9527            busy_refusals: None,
9528            target: RouteTarget::ToolProvider {
9529                module_id: module_id.to_string(),
9530            },
9531            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9532            consumer_identity,
9533            consumer_capabilities: None,
9534            role_versions: None,
9535            admission_facts: facts,
9536            scope: None,
9537        })
9538        .unwrap();
9539        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9540    }
9541
9542    #[derive(Clone, Default)]
9543    struct EventCapture {
9544        events: Arc<Mutex<Vec<CapturedEvent>>>,
9545    }
9546
9547    #[derive(Clone, Debug)]
9548    struct CapturedEvent {
9549        target: String,
9550        level: tracing::Level,
9551        fields: BTreeMap<String, String>,
9552    }
9553
9554    impl EventCapture {
9555        fn events(&self) -> Vec<CapturedEvent> {
9556            self.events.lock().unwrap().clone()
9557        }
9558    }
9559
9560    impl<S> Layer<S> for EventCapture
9561    where
9562        S: Subscriber,
9563    {
9564        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9565            let mut visitor = EventFieldVisitor::default();
9566            event.record(&mut visitor);
9567            self.events.lock().unwrap().push(CapturedEvent {
9568                target: event.metadata().target().to_string(),
9569                level: *event.metadata().level(),
9570                fields: visitor.fields,
9571            });
9572        }
9573    }
9574
9575    #[derive(Default)]
9576    struct EventFieldVisitor {
9577        fields: BTreeMap<String, String>,
9578    }
9579
9580    impl Visit for EventFieldVisitor {
9581        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9582            self.fields
9583                .insert(field.name().to_string(), format!("{value:?}"));
9584        }
9585    }
9586
9587    fn health_response(corr: u64, status: HealthStatus) -> Frame {
9588        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9589            status,
9590            detail: Some("warming".to_string()),
9591            metrics: Some(json!({"queue_depth": 3})),
9592        })
9593        .unwrap();
9594        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9595    }
9596
9597    fn route_bind_ack(corr: u64) -> Frame {
9598        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9599        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9600    }
9601
9602    fn unique_project_root(label: &str) -> ScratchDir {
9603        ScratchDir::new(label)
9604    }
9605
9606    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9607        match parse_route_poll(frame) {
9608            ClientControlResponse::RoutePoll {
9609                status: None,
9610                live: Some(live),
9611                ..
9612            } => assert_eq!(live, expected_live),
9613            other => panic!("unexpected route.poll response: {other:?}"),
9614        }
9615    }
9616
9617    fn bind_liveness_route(
9618        registry: &Registry,
9619        forwarding: &ForwardingTable,
9620        module_id: &str,
9621    ) -> (RouteCtx, u16, u32) {
9622        let module_connection = ConnectionId::new(101);
9623        let client_connection = ConnectionId::new(202);
9624        let registration = registry
9625            .register_with_control_ops(
9626                manifest(module_id, PROTOCOL_VERSION),
9627                PROTOCOL_VERSION,
9628                module_connection,
9629                module_baseline_control_ops(),
9630            )
9631            .unwrap();
9632        let (module_tx, _module_rx) = mpsc::channel(8);
9633        let endpoint = forwarding
9634            .register_module_connection(
9635                module_connection,
9636                module_id.to_string(),
9637                PROTOCOL_VERSION,
9638                manifest_concurrency(&registration.manifest),
9639                FrameSink::new(module_tx),
9640            )
9641            .unwrap();
9642        let (client_ctx, _client_rx) = route_ctx(client_connection);
9643        let pending = forwarding
9644            .begin_route_bind_relay_for_test(
9645                client_connection,
9646                client_ctx.egress.clone(),
9647                1,
9648                module_id,
9649            )
9650            .unwrap();
9651        assert_eq!(pending.endpoint, endpoint);
9652        let route_channel = pending.client_channel;
9653        let route_epoch = pending.client_epoch;
9654        forwarding
9655            .complete_pending_relay(
9656                module_connection,
9657                pending.corr,
9658                RouteBindRelayOutcome::Accepted,
9659            )
9660            .unwrap();
9661        (client_ctx, route_channel, route_epoch)
9662    }
9663
9664    struct FakeProcessLiveness {
9665        live: Option<bool>,
9666    }
9667
9668    impl ModuleProcessLiveness for FakeProcessLiveness {
9669        fn process_live(&self, _module_id: &str) -> Option<bool> {
9670            self.live
9671        }
9672    }
9673
9674    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9675    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9676    {
9677        let registry = Arc::new(Registry::default());
9678        let supervisor_handle = SupervisorHandle::new();
9679        let supervisor = Supervisor::new_for_test(
9680            Arc::clone(&registry),
9681            RestartPolicy::new(1, Duration::from_millis(10)),
9682        )
9683        .with_handle(supervisor_handle.clone());
9684        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9685        let module = supervisor
9686            .spawn(ModuleSpec {
9687                carry_containment: false,
9688                module_id: "stderr-tail-wire".to_string(),
9689                program: fake_aft_stub_path(),
9690                args: Vec::new(),
9691                env: vec![
9692                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9693                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9694                ],
9695                reserved: false,
9696                reserved_prefixes: Vec::new(),
9697                protocol: ModuleProtocol::Subc,
9698                overlap: Default::default(),
9699            })
9700            .unwrap();
9701
9702        let deadline = Instant::now() + Duration::from_secs(5);
9703        loop {
9704            let tail = module.stderr_tail(None, None);
9705            if tail
9706                .entries
9707                .iter()
9708                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9709                && tail.entries.iter().any(|entry| {
9710                    matches!(
9711                        entry,
9712                        TailEntry::Line {
9713                            truncated: true,
9714                            ..
9715                        }
9716                    )
9717                })
9718            {
9719                break;
9720            }
9721            assert!(
9722                Instant::now() < deadline,
9723                "module did not produce a truncated line and restart boundary: {tail:?}"
9724            );
9725            sleep(Duration::from_millis(10)).await;
9726        }
9727
9728        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9729        let request = ClientControlRequest::SupervisorStderrTail {
9730            module_id: "stderr-tail-wire".to_string(),
9731            max_lines: None,
9732            max_bytes: None,
9733        };
9734        let frame = Frame::build(
9735            FrameType::Request,
9736            control_flags(),
9737            0,
9738            0,
9739            1,
9740            serde_json::to_vec(&request).unwrap(),
9741        )
9742        .unwrap();
9743        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9744        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9745        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9746            serde_json::from_slice(&responses[0].body).unwrap()
9747        else {
9748            panic!("expected supervisor.stderr_tail response");
9749        };
9750
9751        assert!(
9752            tail.entries
9753                .iter()
9754                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9755            "the control response lost the restart boundary"
9756        );
9757        let Some(StderrTailEntry::Line {
9758            text,
9759            truncated,
9760            at_ms,
9761        }) = tail.entries.iter().find(|entry| {
9762            matches!(
9763                entry,
9764                StderrTailEntry::Line {
9765                    truncated: true,
9766                    ..
9767                }
9768            )
9769        })
9770        else {
9771            panic!("the control response lost the truncated line");
9772        };
9773        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9774        assert!(*truncated);
9775        assert!(
9776            at_ms.is_some(),
9777            "the control response lost the line's capture time"
9778        );
9779    }
9780
9781    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9782    /// read done on the worker thread would stall every other task until it
9783    /// finished; the read must run off the worker so this test's own task keeps
9784    /// running while the read is paused.
9785    #[tokio::test(flavor = "current_thread")]
9786    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9787        let dir = ScratchDir::new("terminals-off-worker");
9788        let journal_path = dir.join("terminals.jsonl");
9789        let registry = Arc::new(Registry::default());
9790        let supervisor_handle = SupervisorHandle::new();
9791        let supervisor =
9792            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9793                .with_handle(supervisor_handle.clone())
9794                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9795        let module = supervisor
9796            .spawn(ModuleSpec {
9797                carry_containment: false,
9798                module_id: "terminal-off-worker".to_string(),
9799                program: fake_aft_stub_path(),
9800                args: Vec::new(),
9801                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9802                reserved: false,
9803                reserved_prefixes: Vec::new(),
9804                protocol: ModuleProtocol::Subc,
9805                overlap: Default::default(),
9806            })
9807            .unwrap();
9808        let deadline = Instant::now() + Duration::from_secs(5);
9809        while module.terminal_history().entries.len() != 2 {
9810            assert!(Instant::now() < deadline, "module did not record two exits");
9811            sleep(Duration::from_millis(10)).await;
9812        }
9813
9814        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9815        let handler =
9816            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9817        let frame = Frame::build(
9818            FrameType::Request,
9819            control_flags(),
9820            0,
9821            0,
9822            1,
9823            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9824                module_id: "terminal-off-worker".to_string(),
9825            })
9826            .unwrap(),
9827        )
9828        .unwrap();
9829        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9830        let spawned_at = std::time::Instant::now();
9831        let read = tokio::spawn({
9832            let handler = Arc::clone(&handler);
9833            async move { handler.handle_control_frame(&ctx, frame).await }
9834        });
9835        // Waiting for the pause from a blocking thread keeps this task pending,
9836        // so the runtime's single worker is free to run the read task.
9837        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9838            .await
9839            .unwrap()
9840            .expect("the history read reached its pause");
9841        let elapsed = spawned_at.elapsed();
9842        assert!(
9843            elapsed < Duration::from_secs(2) && !read.is_finished(),
9844            "this task could not run while the history read was paused \
9845             (resumed after {elapsed:?}, read finished: {})",
9846            read.is_finished()
9847        );
9848
9849        drop(release);
9850        let responses = read.await.unwrap().unwrap();
9851        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9852        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9853            panic!("expected supervisor.terminals response");
9854        };
9855        assert_eq!(terminals.entries.len(), 2);
9856        assert_eq!(terminals.journal_skipped_lines, 0);
9857        assert_eq!(terminals.journal_read_errors, 0);
9858    }
9859
9860    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9861    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9862        let registry = Arc::new(Registry::default());
9863        let supervisor_handle = SupervisorHandle::new();
9864        let supervisor =
9865            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9866                .with_handle(supervisor_handle.clone());
9867        let module = supervisor
9868            .spawn(ModuleSpec {
9869                carry_containment: false,
9870                module_id: "terminal-golden".to_string(),
9871                program: fake_aft_stub_path(),
9872                args: Vec::new(),
9873                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9874                reserved: false,
9875                reserved_prefixes: Vec::new(),
9876                protocol: ModuleProtocol::Subc,
9877                overlap: Default::default(),
9878            })
9879            .unwrap();
9880
9881        let deadline = Instant::now() + Duration::from_secs(5);
9882        while module.terminal_history().entries.len() != 2 {
9883            assert!(
9884                Instant::now() < deadline,
9885                "module did not retain two terminal exits: {:?}",
9886                module.terminal_history()
9887            );
9888            sleep(Duration::from_millis(10)).await;
9889        }
9890
9891        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9892        let request = ClientControlRequest::SupervisorTerminals {
9893            module_id: "terminal-golden".to_string(),
9894        };
9895        let frame = Frame::build(
9896            FrameType::Request,
9897            control_flags(),
9898            0,
9899            0,
9900            1,
9901            serde_json::to_vec(&request).unwrap(),
9902        )
9903        .unwrap();
9904        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9905        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9906        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9907        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9908            panic!("expected supervisor.terminals response");
9909        };
9910        assert_eq!(terminals.entries.len(), 2);
9911        assert_eq!(terminals.dropped, 0);
9912
9913        let mut rendered = serde_json::to_value(response).unwrap();
9914        // Wall-clock fields are the observation contract, but not stable fixture
9915        // bytes; normalize only them after the real handler has shaped the response.
9916        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9917        for (index, entry) in rendered["entries"]
9918            .as_array_mut()
9919            .expect("terminal response entries array")
9920            .iter_mut()
9921            .enumerate()
9922        {
9923            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9924        }
9925
9926        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9927            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9928        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9929        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9930            std::fs::write(&golden_path, &serialized).unwrap();
9931        }
9932        let expected: Value =
9933            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9934        assert_eq!(rendered, expected);
9935    }
9936
9937    #[test]
9938    fn hello_registers_manifest_and_returns_ack() {
9939        let registry = Arc::new(Registry::default());
9940        let handler = ControlHandler::new(Arc::clone(&registry));
9941        let conn = ConnectionId::new(1);
9942
9943        let responses = handler
9944            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9945            .unwrap();
9946
9947        assert_eq!(responses.len(), 1);
9948        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9949        assert_eq!(responses[0].header.channel, 0);
9950        assert_eq!(responses[0].header.corr, 7);
9951        let ack = parse_ack(&responses[0]);
9952        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9953        assert!(ack
9954            .subc_capabilities
9955            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9956        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9957        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9958        assert!(ack
9959            .subc_ops
9960            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9961        assert!(ack
9962            .subc_ops
9963            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9964
9965        let registration = registry.get_module("aft").unwrap().unwrap();
9966        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9967        assert_eq!(registration.state, ChannelState::Active);
9968        assert_eq!(registration.connection_id, conn);
9969        assert_eq!(registration.control_ops, module_baseline_control_ops());
9970        assert!(registration.manifest.display_name.is_none());
9971    }
9972
9973    #[test]
9974    fn hello_display_name_refuses_invalid_manifest_by_field() {
9975        let registry = Arc::new(Registry::default());
9976        let handler = ControlHandler::new(Arc::clone(&registry));
9977        for (index, invalid) in [
9978            json!(""),
9979            json!("a".repeat(41)),
9980            json!(" Routing"),
9981            json!("a\nb"),
9982            json!("a\u{202e}b"),
9983            json!(7),
9984        ]
9985        .into_iter()
9986        .enumerate()
9987        {
9988            let module_id = format!("bad-display-name-{index}");
9989            let mut body = serde_json::to_value(ModuleHelloBody {
9990                manifest: manifest(&module_id, PROTOCOL_VERSION),
9991                protocol_ver: PROTOCOL_VERSION,
9992                control_ops: None,
9993                launch_nonce: None,
9994            })
9995            .unwrap();
9996            body["manifest"]["display_name"] = invalid;
9997            let hello = Frame::build(
9998                FrameType::Hello,
9999                control_flags(),
10000                0,
10001                0,
10002                7,
10003                serde_json::to_vec(&body).unwrap(),
10004            )
10005            .unwrap();
10006            let response = handler
10007                .handle_control(ConnectionId::new(100 + index as u64), hello)
10008                .unwrap();
10009            assert_eq!(response[0].header.ty, FrameType::Error);
10010            let error = parse_error(&response[0]);
10011            assert_eq!(error["code"], "invalid_manifest");
10012            assert!(error["message"].as_str().unwrap().contains("display_name"));
10013            assert!(registry.get_module(&module_id).unwrap().is_none());
10014        }
10015    }
10016
10017    #[test]
10018    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10019        let invalid_identifiers = [
10020            ("case_change", "credentials-Provider/v1"),
10021            ("leading_zero", "credentials-provider/v01"),
10022            ("trailing_hyphen", "credentials-provider-/v1"),
10023            ("consecutive_hyphens", "credentials--provider/v1"),
10024            ("uppercase", "Credentials-provider/v1"),
10025            ("missing_v", "credentials-provider/1"),
10026            ("whitespace", "credentials provider/v1"),
10027            ("zero_version", "credentials-provider/v0"),
10028            ("out_of_range_version", "credentials-provider/v4294967296"),
10029            (
10030                "overlength_name",
10031                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10032            ),
10033        ];
10034        let mut cases = invalid_identifiers
10035            .into_iter()
10036            .map(|(name, identifier)| {
10037                (
10038                    format!("identifier_{name}"),
10039                    "capabilities.provides[0]".to_string(),
10040                    identifier.to_string(),
10041                    json!({ "provides": [identifier] }),
10042                    None,
10043                )
10044            })
10045            .collect::<Vec<_>>();
10046        cases.extend([
10047            (
10048                "unknown_need".to_string(),
10049                "capabilities.requires[0].need".to_string(),
10050                "deferred".to_string(),
10051                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10052                None,
10053            ),
10054            (
10055                "duplicate_provides".to_string(),
10056                "capabilities.provides[1]".to_string(),
10057                "credentials-provider/v1".to_string(),
10058                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10059                None,
10060            ),
10061            (
10062                "duplicate_must_never_reach".to_string(),
10063                "capabilities.must_never_reach[1]".to_string(),
10064                "credentials-provider/v1".to_string(),
10065                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10066                None,
10067            ),
10068            (
10069                "duplicate_requires_same_need".to_string(),
10070                "capabilities.requires[1]".to_string(),
10071                "credentials-provider/v1".to_string(),
10072                json!({ "requires": [
10073                    { "capability": "credentials-provider/v1", "need": "required" },
10074                    { "capability": "credentials-provider/v1", "need": "required" }
10075                ] }),
10076                None,
10077            ),
10078            (
10079                "duplicate_requires_conflicting_need".to_string(),
10080                "capabilities.requires[1]".to_string(),
10081                "credentials-provider/v1".to_string(),
10082                json!({ "requires": [
10083                    { "capability": "credentials-provider/v1", "need": "required" },
10084                    { "capability": "credentials-provider/v1", "need": "optional" }
10085                ] }),
10086                None,
10087            ),
10088            (
10089                "capabilities_root_pointer".to_string(),
10090                "runtime_computed[0]".to_string(),
10091                "/capabilities".to_string(),
10092                json!({}),
10093                Some(json!(["/capabilities"])),
10094            ),
10095            (
10096                "capabilities_descendant_pointer".to_string(),
10097                "runtime_computed[0]".to_string(),
10098                "/capabilities/provides".to_string(),
10099                json!({}),
10100                Some(json!(["/capabilities/provides"])),
10101            ),
10102            (
10103                "malformed_pointer_without_leading_slash".to_string(),
10104                "runtime_computed[0]".to_string(),
10105                "capabilities".to_string(),
10106                json!({}),
10107                Some(json!(["capabilities"])),
10108            ),
10109            (
10110                "malformed_pointer_escape".to_string(),
10111                "runtime_computed[0]".to_string(),
10112                "/roles/~2/tools".to_string(),
10113                json!({}),
10114                Some(json!(["/roles/~2/tools"])),
10115            ),
10116            (
10117                "unknown_capabilities_field".to_string(),
10118                "capabilities.future".to_string(),
10119                "<array>".to_string(),
10120                json!({ "future": [] }),
10121                None,
10122            ),
10123        ]);
10124
10125        for (index, (name, field, value, capabilities, runtime_computed)) in
10126            cases.into_iter().enumerate()
10127        {
10128            let registry = Arc::new(Registry::default());
10129            let handler = ControlHandler::new(Arc::clone(&registry));
10130            let response = handler
10131                .handle_control(
10132                    ConnectionId::new((index + 1) as u64),
10133                    capability_grammar_hello_frame(
10134                        capabilities,
10135                        runtime_computed,
10136                        index as u64 + 1,
10137                    ),
10138                )
10139                .expect("invalid HELLO returns a refusal");
10140
10141            assert_eq!(response.len(), 1, "{name} must emit one refusal");
10142            let error = parse_error(&response[0]);
10143            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10144            let message = error["message"]
10145                .as_str()
10146                .expect("error message is a string");
10147            assert!(
10148                message.contains(&field),
10149                "{name}: field missing from {message}"
10150            );
10151            assert!(
10152                message.contains(&value),
10153                "{name}: value missing from {message}"
10154            );
10155            assert_eq!(
10156                registry
10157                    .active_registration_count()
10158                    .expect("registry reads"),
10159                0,
10160                "{name}: refused HELLO must not create a catalog entry"
10161            );
10162        }
10163    }
10164
10165    #[test]
10166    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10167        let registry = Arc::new(Registry::default());
10168        let handler = ControlHandler::new(Arc::clone(&registry));
10169        let capabilities = json!({
10170            "provides": ["credentials-provider/v1"],
10171            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10172            "must_never_reach": ["federation-transport/v1"]
10173        });
10174        let response = handler
10175            .handle_control(
10176                ConnectionId::new(99),
10177                capability_grammar_hello_frame(
10178                    capabilities.clone(),
10179                    Some(json!(["/roles/0/tools"])),
10180                    99,
10181                ),
10182            )
10183            .expect("valid HELLO registers");
10184        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10185
10186        let request = Frame::build(
10187            FrameType::Request,
10188            control_flags(),
10189            0,
10190            0,
10191            100,
10192            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10193                .expect("catalog request serializes"),
10194        )
10195        .expect("catalog request frame builds");
10196        let response = handler
10197            .handle_catalog_list(request, None)
10198            .expect("catalog list succeeds");
10199        let ClientControlResponse::CatalogList { modules, .. } =
10200            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10201        else {
10202            panic!("catalog request must return catalog.list");
10203        };
10204        assert_eq!(modules.len(), 1);
10205        assert_eq!(
10206            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10207            capabilities
10208        );
10209    }
10210
10211    #[test]
10212    fn catalog_list_mirrors_management_operation_description() {
10213        let registry = Arc::new(Registry::default());
10214        let handler = ControlHandler::new(Arc::clone(&registry));
10215        let description = "List managed records and return their identifiers and metadata.";
10216        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10217        manifest.provides = vec![ProviderRole::ManagementSurface {
10218            operations: vec![ManagementOperation {
10219                name: "records.list".to_string(),
10220                kind: ManagementOperationKind::Query,
10221                description: Some(description.to_string()),
10222            }],
10223            config_schema: json!({"type": "object"}),
10224            observability: vec![ObservabilitySurface {
10225                name: "records.stats".to_string(),
10226                kind: ObservabilityKind::Snapshot,
10227            }],
10228            identity_scope: vec![IdentityScope::Project],
10229            concurrency: Concurrency::ModuleManaged,
10230        }];
10231        registry
10232            .register_with_control_ops(
10233                manifest,
10234                PROTOCOL_VERSION,
10235                ConnectionId::new(99),
10236                Vec::new(),
10237            )
10238            .expect("described management manifest registers");
10239
10240        let request = Frame::build(
10241            FrameType::Request,
10242            control_flags(),
10243            0,
10244            0,
10245            100,
10246            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10247                .expect("catalog request serializes"),
10248        )
10249        .expect("catalog request frame builds");
10250        let response = handler
10251            .handle_catalog_list(request, None)
10252            .expect("catalog list succeeds");
10253        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10254        assert_eq!(
10255            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10256            "catalog.list must preserve the declared operation description verbatim"
10257        );
10258    }
10259
10260    #[test]
10261    fn catalog_list_mirrors_registered_operation_delivery() {
10262        let registry = Arc::new(Registry::default());
10263        let handler = ControlHandler::new(Arc::clone(&registry));
10264        let delivery =
10265            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10266        let mut body = serde_json::to_value(ModuleHelloBody {
10267            manifest: manifest("watch-management", PROTOCOL_VERSION),
10268            protocol_ver: PROTOCOL_VERSION,
10269            control_ops: None,
10270            launch_nonce: None,
10271        })
10272        .unwrap();
10273        body["manifest"]["provides"] = json!([{
10274            "role": "management_surface",
10275            "operations": [
10276                {"name": "status", "kind": "query"},
10277                {"name": "future", "kind": "query"},
10278                {"name": "reset", "kind": "mutate"}
10279            ],
10280            "config_schema": {}, "observability": [], "identity_scope": []
10281        }]);
10282        body["manifest"]["operation_delivery"] = delivery.clone();
10283        let hello = Frame::build(
10284            FrameType::Hello,
10285            control_flags(),
10286            0,
10287            0,
10288            99,
10289            serde_json::to_vec(&body).unwrap(),
10290        )
10291        .unwrap();
10292        let response = handler
10293            .handle_control(ConnectionId::new(99), hello)
10294            .unwrap();
10295        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10296
10297        let (_, registrations) = registry.list_modules().unwrap();
10298        let registered = &registrations[0].manifest;
10299        assert!(registered.is_watch_operation("status"));
10300        assert!(!registered.is_watch_operation("future"));
10301        assert!(!registered.is_watch_operation("reset"));
10302        assert!(!registered.is_watch_operation("missing"));
10303
10304        let request = Frame::build(
10305            FrameType::Request,
10306            control_flags(),
10307            0,
10308            0,
10309            100,
10310            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10311        )
10312        .unwrap();
10313        let response = handler.handle_catalog_list(request, None).unwrap();
10314        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10315        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10316        let ClientControlResponse::CatalogList { modules, .. } =
10317            serde_json::from_slice(&response[0].body).unwrap()
10318        else {
10319            panic!("expected catalog.list");
10320        };
10321        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10322    }
10323
10324    #[test]
10325    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10326        let registry = Arc::new(Registry::default());
10327        let supervisor = SupervisorHandle::new();
10328        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10329        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10330        let descriptor = json!({
10331            "schema": "ckios.richui.manifest/1",
10332            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10333            "bytes": 4096,
10334            "chunks": 4
10335        });
10336        let connection = ConnectionId::new(201);
10337        let hello = handler
10338            .handle_control(
10339                connection,
10340                hello_frame_with_rich_ui(
10341                    "vault",
10342                    PROTOCOL_VERSION,
10343                    201,
10344                    Some("the-real-nonce"),
10345                    descriptor.clone(),
10346                ),
10347            )
10348            .unwrap();
10349        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10350
10351        let request =
10352            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10353        let provides = registry
10354            .get_module_by_connection(connection)
10355            .unwrap()
10356            .unwrap()
10357            .manifest
10358            .provides;
10359        handler
10360            .handle_catalog_update(connection, request, provides, None, Some(false))
10361            .unwrap();
10362
10363        let request = Frame::build(
10364            FrameType::Request,
10365            control_flags(),
10366            0,
10367            0,
10368            203,
10369            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10370        )
10371        .unwrap();
10372        let response = handler.handle_catalog_list(request, None).unwrap();
10373        let ClientControlResponse::CatalogList { modules, .. } =
10374            serde_json::from_slice(&response[0].body).unwrap()
10375        else {
10376            panic!("expected catalog.list");
10377        };
10378        assert_eq!(
10379            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10380            descriptor
10381        );
10382        assert!(modules[0].nonce_verified);
10383    }
10384
10385    fn hello_with_role_declarations(declarations: Value) -> Frame {
10386        let mut body = serde_json::to_value(ModuleHelloBody {
10387            manifest: manifest("role-test", PROTOCOL_VERSION),
10388            protocol_ver: PROTOCOL_VERSION,
10389            control_ops: None,
10390            launch_nonce: None,
10391        })
10392        .unwrap();
10393        body["manifest"]["role_declarations"] = declarations;
10394        Frame::build(
10395            FrameType::Hello,
10396            control_flags(),
10397            0,
10398            0,
10399            301,
10400            serde_json::to_vec(&body).unwrap(),
10401        )
10402        .unwrap()
10403    }
10404
10405    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10406        let request = Frame::build(
10407            FrameType::Request,
10408            control_flags(),
10409            0,
10410            0,
10411            302,
10412            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10413        )
10414        .unwrap();
10415        let response = handler.handle_catalog_list(request, None).unwrap();
10416        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10417        let ClientControlResponse::CatalogList { mut modules, .. } =
10418            serde_json::from_slice(&response[0].body).unwrap()
10419        else {
10420            panic!("expected catalog.list");
10421        };
10422        (raw["modules"][0].clone(), modules.remove(0))
10423    }
10424
10425    #[test]
10426    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10427        let registry = Arc::new(Registry::default());
10428        let handler = ControlHandler::new(Arc::clone(&registry));
10429        let declarations = json!({
10430            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10431            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10432        });
10433        let connection = ConnectionId::new(301);
10434        let response = handler
10435            .handle_control(
10436                connection,
10437                hello_with_role_declarations(declarations.clone()),
10438            )
10439            .unwrap();
10440        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10441        let registered = registry
10442            .get_module_by_connection(connection)
10443            .unwrap()
10444            .unwrap()
10445            .manifest;
10446        // Compared as JSON values, not bytes: object key order is not part of
10447        // the relay's promise, and workspace builds unify serde_json's
10448        // preserve_order feature, which changes the order a literal keeps.
10449        assert_eq!(
10450            serde_json::to_value(&registered.role_declarations).unwrap(),
10451            declarations
10452        );
10453        let (raw, entry) = role_declarations_catalog(&handler);
10454        assert_eq!(raw["role_declarations"], declarations);
10455        assert_eq!(entry.role_declarations, registered.role_declarations);
10456
10457        let request =
10458            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10459        handler
10460            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10461            .unwrap();
10462        let (raw, entry) = role_declarations_catalog(&handler);
10463        assert_eq!(raw["role_declarations"], declarations);
10464        assert_eq!(entry.role_declarations, registered.role_declarations);
10465    }
10466
10467    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10468        let registry = Arc::new(Registry::default());
10469        let handler = ControlHandler::new(Arc::clone(&registry));
10470        let capture = EventCapture::default();
10471        let _subscriber =
10472            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10473        let response = handler
10474            .handle_control(
10475                ConnectionId::new(301),
10476                hello_with_role_declarations(declarations),
10477            )
10478            .unwrap();
10479        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10480        assert!(registry
10481            .get_module_by_connection(ConnectionId::new(301))
10482            .unwrap()
10483            .unwrap()
10484            .manifest
10485            .role_declarations
10486            .is_empty());
10487        let (raw, entry) = role_declarations_catalog(&handler);
10488        assert!(raw.get("role_declarations").is_none());
10489        assert!(entry.role_declarations.is_empty());
10490        let warnings: Vec<_> = capture
10491            .events()
10492            .into_iter()
10493            .filter(|event| {
10494                event.fields.get("message").map(String::as_str)
10495                    == Some("dropping whole role_declarations map from HELLO")
10496            })
10497            .collect();
10498        assert_eq!(warnings.len(), 1);
10499        assert_eq!(warnings[0].level, tracing::Level::WARN);
10500        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10501        assert_eq!(warnings[0].fields["reason"], reason);
10502    }
10503
10504    #[test]
10505    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10506        for value in [
10507            json!(null),
10508            json!(false),
10509            json!(0),
10510            json!("opaque"),
10511            json!([]),
10512        ] {
10513            assert_hello_drops_role_declarations_with_warning(
10514                json!({"routing/v1": {}, "broken/v1": value}),
10515                "role_declarations values must be objects",
10516            );
10517        }
10518        assert_hello_drops_role_declarations_with_warning(
10519            json!([]),
10520            "role_declarations must be an object",
10521        );
10522    }
10523
10524    #[test]
10525    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10526        assert_hello_drops_role_declarations_with_warning(
10527            json!({"routing/v1": {}, "bad key": {}}),
10528            "role_declarations key must use capability-name grammar",
10529        );
10530    }
10531
10532    #[test]
10533    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10534        assert_hello_drops_role_declarations_with_warning(
10535            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10536            "role_declarations exceeds 16384 compact JSON bytes",
10537        );
10538    }
10539
10540    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10541        let mut body = serde_json::to_value(ModuleHelloBody {
10542            manifest: manifest("flow-ops", PROTOCOL_VERSION),
10543            protocol_ver: PROTOCOL_VERSION,
10544            control_ops: None,
10545            launch_nonce: None,
10546        })
10547        .unwrap();
10548        body["manifest"]["provides"] = json!([{
10549            "role": "management_surface",
10550            "operations": [
10551                {"name": "diagnostics.findings", "kind": "query"},
10552                {"name": "reset", "kind": "mutate"}
10553            ],
10554            "config_schema": {}, "observability": [], "identity_scope": []
10555        }]);
10556        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10557        if let Some(flow_operations) = flow_operations {
10558            body["manifest"]["flow_operations"] = flow_operations;
10559        }
10560        Frame::build(
10561            FrameType::Hello,
10562            control_flags(),
10563            0,
10564            0,
10565            401,
10566            serde_json::to_vec(&body).unwrap(),
10567        )
10568        .unwrap()
10569    }
10570
10571    #[test]
10572    fn catalog_list_mirrors_flow_operations_unchanged() {
10573        // Order is kept as declared (not sorted) and an empty list stays an
10574        // affirmative "none", distinct from an absent declaration.
10575        for (capability, flow_operations) in [
10576            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10577            ("call-modes/v1", json!(["diagnostics.findings"])),
10578            ("flow-scopes/v1", json!([])),
10579        ] {
10580            let registry = Arc::new(Registry::default());
10581            let handler = ControlHandler::new(Arc::clone(&registry));
10582            let connection = ConnectionId::new(401);
10583            let response = handler
10584                .handle_control(
10585                    connection,
10586                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10587                )
10588                .unwrap();
10589            assert_eq!(response[0].header.ty, FrameType::HelloAck);
10590            let registered = registry
10591                .get_module_by_connection(connection)
10592                .unwrap()
10593                .unwrap()
10594                .manifest;
10595            assert_eq!(
10596                serde_json::to_value(&registered.flow_operations).unwrap(),
10597                flow_operations
10598            );
10599            let (raw, entry) = role_declarations_catalog(&handler);
10600            assert_eq!(raw["flow_operations"], flow_operations);
10601            assert_eq!(entry.flow_operations, registered.flow_operations);
10602        }
10603
10604        let registry = Arc::new(Registry::default());
10605        let handler = ControlHandler::new(Arc::clone(&registry));
10606        let response = handler
10607            .handle_control(
10608                ConnectionId::new(401),
10609                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10610            )
10611            .unwrap();
10612        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10613        let (raw, entry) = role_declarations_catalog(&handler);
10614        assert!(raw.get("flow_operations").is_none());
10615        assert_eq!(entry.flow_operations, None);
10616    }
10617
10618    fn assert_hello_drops_flow_operations_with_warning(
10619        capabilities: Value,
10620        flow_operations: Value,
10621        reason: &str,
10622    ) {
10623        let registry = Arc::new(Registry::default());
10624        let handler = ControlHandler::new(Arc::clone(&registry));
10625        let capture = EventCapture::default();
10626        let _subscriber =
10627            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10628        let response = handler
10629            .handle_control(
10630                ConnectionId::new(401),
10631                hello_with_flow_operations(capabilities, Some(flow_operations)),
10632            )
10633            .unwrap();
10634        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10635        let registered = registry
10636            .get_module_by_connection(ConnectionId::new(401))
10637            .unwrap()
10638            .expect("the module still registers")
10639            .manifest;
10640        assert_eq!(registered.module_id, "flow-ops");
10641        assert_eq!(registered.flow_operations, None);
10642        let (raw, entry) = role_declarations_catalog(&handler);
10643        assert_eq!(raw["module_id"], "flow-ops");
10644        assert!(raw.get("flow_operations").is_none());
10645        assert_eq!(entry.flow_operations, None);
10646        let warnings: Vec<_> = capture
10647            .events()
10648            .into_iter()
10649            .filter(|event| {
10650                event.fields.get("message").map(String::as_str)
10651                    == Some("dropping flow_operations from HELLO")
10652            })
10653            .collect();
10654        assert_eq!(warnings.len(), 1);
10655        assert_eq!(warnings[0].level, tracing::Level::WARN);
10656        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10657        assert_eq!(warnings[0].fields["reason"], reason);
10658    }
10659
10660    #[test]
10661    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10662        assert_hello_drops_flow_operations_with_warning(
10663            json!(["flow-scopes/v1"]),
10664            json!(["diagnostics.findings", "missing"]),
10665            "flow_operations names 'missing', which is not a declared management operation",
10666        );
10667    }
10668
10669    #[test]
10670    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10671        assert_hello_drops_flow_operations_with_warning(
10672            json!([]),
10673            json!(["diagnostics.findings"]),
10674            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10675        );
10676        // A run-only capability does not admit flows either.
10677        assert_hello_drops_flow_operations_with_warning(
10678            json!(["agent-run-scopes/v1"]),
10679            json!([]),
10680            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10681        );
10682    }
10683
10684    #[test]
10685    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10686        assert_hello_drops_flow_operations_with_warning(
10687            json!(["flow-scopes/v1"]),
10688            json!(["reset", "reset"]),
10689            "flow_operations names 'reset' more than once",
10690        );
10691        assert_hello_drops_flow_operations_with_warning(
10692            json!(["flow-scopes/v1"]),
10693            json!(vec!["reset"; 257]),
10694            "flow_operations lists 257 names, more than 256",
10695        );
10696        assert_hello_drops_flow_operations_with_warning(
10697            json!(["flow-scopes/v1"]),
10698            json!("reset"),
10699            "flow_operations must be an array of operation names",
10700        );
10701    }
10702
10703    #[test]
10704    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10705        let registry = Arc::new(Registry::default());
10706        let handler = ControlHandler::new(registry.clone());
10707        let descriptor = json!({
10708            "schema": "ckios.richui.manifest/1",
10709            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10710            "bytes": 128,
10711            "chunks": 2
10712        });
10713        let malformed = [
10714            ("schema", json!("")),
10715            ("schema", json!("a".repeat(65))),
10716            ("schema", json!("Bad schema")),
10717            (
10718                "manifest_digest",
10719                json!(format!("sha256:{}", "A".repeat(64))),
10720            ),
10721            ("manifest_digest", json!("sha256:abc")),
10722            ("bytes", json!(0)),
10723            ("bytes", json!(16_777_217)),
10724            ("bytes", json!("128")),
10725            ("chunks", json!(0)),
10726            ("chunks", json!(1025)),
10727            ("chunks", json!("2")),
10728        ];
10729        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10730            let mut invalid = descriptor.clone();
10731            invalid[field] = invalid_value;
10732            let response = handler
10733                .handle_control(
10734                    ConnectionId::new(210 + index as u64),
10735                    hello_frame_with_rich_ui(
10736                        &format!("bad-rich-ui-{index}"),
10737                        PROTOCOL_VERSION,
10738                        210 + index as u64,
10739                        None,
10740                        invalid,
10741                    ),
10742                )
10743                .unwrap();
10744            assert_eq!(response[0].header.ty, FrameType::Error);
10745            let error = parse_error(&response[0]);
10746            assert_eq!(error["code"], "invalid_manifest");
10747            assert!(error["message"]
10748                .as_str()
10749                .unwrap()
10750                .contains(&format!("rich_ui.{field}")));
10751            assert!(registry
10752                .get_module(&format!("bad-rich-ui-{index}"))
10753                .unwrap()
10754                .is_none());
10755        }
10756    }
10757
10758    #[test]
10759    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10760        let registry = Arc::new(Registry::default());
10761        let handler = ControlHandler::new(Arc::clone(&registry));
10762        let connection = ConnectionId::new(219);
10763        registry
10764            .register_with_control_ops(
10765                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10766                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10767                        "bad schema",
10768                        format!("sha256:{}", "a".repeat(64)),
10769                        128,
10770                        2,
10771                    )))
10772                    .build(),
10773                PROTOCOL_VERSION,
10774                connection,
10775                Vec::new(),
10776            )
10777            .unwrap();
10778        let request =
10779            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10780        let response = handler
10781            .handle_catalog_update(connection, request, Vec::new(), None, None)
10782            .unwrap();
10783        assert_eq!(response[0].header.ty, FrameType::Error);
10784        let error = parse_error(&response[0]);
10785        assert_eq!(error["code"], "invalid_manifest");
10786        assert!(error["message"]
10787            .as_str()
10788            .unwrap()
10789            .contains("rich_ui.schema"));
10790    }
10791
10792    #[test]
10793    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10794        let registry = Arc::new(Registry::default());
10795        let supervisor = SupervisorHandle::new();
10796        supervisor.open_swap("provider", "candidate-nonce".to_string());
10797        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10798        let connection = ConnectionId::new(218);
10799        let hello = handler
10800            .handle_control(
10801                connection,
10802                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10803            )
10804            .unwrap();
10805        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10806        assert!(registry.list_modules().unwrap().1.is_empty());
10807
10808        registry.promote_candidate("provider").unwrap().unwrap();
10809        let request = Frame::build(
10810            FrameType::Request,
10811            control_flags(),
10812            0,
10813            0,
10814            219,
10815            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10816        )
10817        .unwrap();
10818        let response = handler.handle_catalog_list(request, None).unwrap();
10819        let ClientControlResponse::CatalogList { modules, .. } =
10820            serde_json::from_slice(&response[0].body).unwrap()
10821        else {
10822            panic!("expected catalog.list");
10823        };
10824        assert_eq!(modules.len(), 1);
10825        assert!(modules[0].nonce_verified);
10826    }
10827
10828    #[test]
10829    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10830        for provenance_json in [
10831            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10832            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10833        ] {
10834            let registry = Arc::new(Registry::default());
10835            let handler = ControlHandler::new(registry);
10836            let hello = Frame::build(
10837                FrameType::Hello,
10838                control_flags(),
10839                0,
10840                0,
10841                220,
10842                serde_json::to_vec(&ModuleHelloBody {
10843                    manifest: ModuleManifest::builder("built", "0.1.0")
10844                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10845                        .build(),
10846                    protocol_ver: PROTOCOL_VERSION,
10847                    control_ops: None,
10848                    launch_nonce: None,
10849                })
10850                .unwrap(),
10851            )
10852            .unwrap();
10853            assert_eq!(
10854                handler
10855                    .handle_control(ConnectionId::new(220), hello)
10856                    .unwrap()[0]
10857                    .header
10858                    .ty,
10859                FrameType::HelloAck
10860            );
10861            let request = Frame::build(
10862                FrameType::Request,
10863                control_flags(),
10864                0,
10865                0,
10866                221,
10867                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10868            )
10869            .unwrap();
10870            let response = handler.handle_catalog_list(request, None).unwrap();
10871            let ClientControlResponse::CatalogList { modules, .. } =
10872                serde_json::from_slice(&response[0].body).unwrap()
10873            else {
10874                panic!("expected catalog.list");
10875            };
10876            assert_eq!(modules.len(), 1);
10877            assert_eq!(
10878                serde_json::to_vec(&modules[0]).unwrap(),
10879                format!(
10880                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10881                )
10882                .as_bytes()
10883            );
10884        }
10885    }
10886
10887    #[test]
10888    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10889        let registry = Arc::new(Registry::default());
10890        let handler = ControlHandler::new(Arc::clone(&registry));
10891        let hello = Frame::build(
10892            FrameType::Hello,
10893            control_flags(),
10894            0,
10895            0,
10896            220,
10897            serde_json::to_vec(&ModuleHelloBody {
10898                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10899                protocol_ver: PROTOCOL_VERSION,
10900                control_ops: None,
10901                launch_nonce: Some("unverified".to_string()),
10902            })
10903            .unwrap(),
10904        )
10905        .unwrap();
10906        assert_eq!(
10907            handler
10908                .handle_control(ConnectionId::new(220), hello)
10909                .unwrap()[0]
10910                .header
10911                .ty,
10912            FrameType::HelloAck
10913        );
10914        let request = Frame::build(
10915            FrameType::Request,
10916            control_flags(),
10917            0,
10918            0,
10919            221,
10920            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10921        )
10922        .unwrap();
10923        let response = handler.handle_catalog_list(request, None).unwrap();
10924        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10925        assert!(wire["modules"][0].get("provenance").is_none());
10926        let ClientControlResponse::CatalogList { modules, .. } =
10927            serde_json::from_slice(&response[0].body).unwrap()
10928        else {
10929            panic!("expected catalog.list");
10930        };
10931        assert_eq!(
10932            serde_json::to_vec(&modules[0]).unwrap(),
10933            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10934        );
10935        assert!(!modules[0].nonce_verified);
10936        assert!(modules[0].provenance.is_none());
10937    }
10938
10939    #[test]
10940    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10941        let registry = Arc::new(Registry::default());
10942        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
10943            [("vault".to_string(), true), ("squatter".to_string(), true)],
10944            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10945        );
10946        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10947        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10948            provides: vec!["credentials-provider/v1".to_string()],
10949            requires: Vec::new(),
10950            must_never_reach: Vec::new(),
10951        });
10952        let frame = Frame::build(
10953            FrameType::Hello,
10954            control_flags(),
10955            0,
10956            0,
10957            77,
10958            serde_json::to_vec(&ModuleHelloBody {
10959                manifest: squatter,
10960                protocol_ver: PROTOCOL_VERSION,
10961                control_ops: None,
10962                launch_nonce: None,
10963            })
10964            .expect("HELLO serializes"),
10965        )
10966        .expect("HELLO frame builds");
10967        let response = handler
10968            .handle_control(ConnectionId::new(77), frame)
10969            .expect("reserved claim receives a typed refusal");
10970        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10971        assert_eq!(
10972            registry
10973                .active_registration_count()
10974                .expect("registry reads"),
10975            0,
10976            "a reserved capability refusal must not leave a catalog entry"
10977        );
10978    }
10979
10980    #[test]
10981    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10982        for settlement in ["timeout", "inconclusive", "drop"] {
10983            let breakers = RouteBindBreakers::default();
10984            let RouteBindAdmission::Admitted {
10985                guard: mut old,
10986                probe: false,
10987            } = breakers.admit("prov")
10988            else {
10989                panic!("ordinary relay admitted")
10990            };
10991            let RouteBindAdmission::Admitted {
10992                guard: mut opener, ..
10993            } = breakers.admit("prov")
10994            else {
10995                panic!("second relay admitted")
10996            };
10997            assert!(
10998                !opener
10999                    .record_timeout(1, Duration::ZERO)
11000                    .unwrap()
11001                    .reopened_after_probe
11002            );
11003            let RouteBindAdmission::Admitted {
11004                guard: mut probe,
11005                probe: true,
11006            } = breakers.admit("prov")
11007            else {
11008                panic!("one cooldown probe admitted")
11009            };
11010            match settlement {
11011                "timeout" => assert!(
11012                    !old.record_timeout(1, Duration::ZERO)
11013                        .unwrap()
11014                        .reopened_after_probe
11015                ),
11016                "inconclusive" => old.record_inconclusive(),
11017                "drop" => drop(old),
11018                _ => unreachable!(),
11019            }
11020            assert!(
11021                matches!(
11022                    breakers.admit("prov"),
11023                    RouteBindAdmission::Refused {
11024                        probe_in_flight: true,
11025                        ..
11026                    }
11027                ),
11028                "{settlement} of a pre-open relay cannot release the real probe"
11029            );
11030            assert!(
11031                probe
11032                    .record_timeout(1, Duration::ZERO)
11033                    .unwrap()
11034                    .reopened_after_probe
11035            );
11036            assert!(matches!(
11037                breakers.admit("prov"),
11038                RouteBindAdmission::Admitted { probe: true, .. }
11039            ));
11040        }
11041        let breakers = RouteBindBreakers::default();
11042        let admit = || match breakers.admit("prov") {
11043            RouteBindAdmission::Admitted { guard, .. } => guard,
11044            _ => panic!("relay admitted"),
11045        };
11046        admit().record_timeout(1, Duration::ZERO);
11047        let mut old_probe = admit();
11048        breakers.reset_for_new_module_connection("prov");
11049        admit().record_timeout(1, Duration::ZERO);
11050        let _new_probe = admit();
11051        old_probe.record_inconclusive();
11052        assert!(matches!(
11053            breakers.admit("prov"),
11054            RouteBindAdmission::Refused {
11055                probe_in_flight: true,
11056                ..
11057            }
11058        ));
11059    }
11060
11061    #[tokio::test]
11062    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11063        for candidate in [false, true] {
11064            let registry = Arc::new(Registry::default());
11065            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
11066                [("vault".to_string(), true), ("squatter".to_string(), true)],
11067                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11068            );
11069            let conn = ConnectionId::new(77);
11070            let (ctx, mut rx) = route_ctx(conn);
11071            let initial = capability_manifest("squatter", &[], &[]);
11072            if candidate {
11073                registry
11074                    .register_candidate_with_control_ops(
11075                        initial.clone(),
11076                        PROTOCOL_VERSION,
11077                        conn,
11078                        module_baseline_control_ops(),
11079                    )
11080                    .unwrap();
11081                handler
11082                    .forwarding
11083                    .register_candidate_module_connection(
11084                        conn,
11085                        "squatter".to_string(),
11086                        PROTOCOL_VERSION,
11087                        manifest_concurrency(&initial),
11088                        ctx.egress.clone(),
11089                    )
11090                    .unwrap();
11091            } else {
11092                hello_via_sink(
11093                    &handler,
11094                    &ctx,
11095                    &mut rx,
11096                    hello_frame_with_manifest(initial.clone(), 1),
11097                )
11098                .await;
11099            }
11100            let response = handler
11101                .handle_control_frame(
11102                    &ctx,
11103                    catalog_update_with_capabilities_frame(
11104                        2,
11105                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
11106                            .capabilities
11107                            .unwrap(),
11108                    ),
11109                )
11110                .await
11111                .unwrap();
11112            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11113            assert_eq!(
11114                registry
11115                    .get_module_by_connection(conn)
11116                    .unwrap()
11117                    .unwrap()
11118                    .manifest,
11119                initial
11120            );
11121        }
11122    }
11123
11124    #[test]
11125    fn server_describe_surfaces_required_capability_verdict_fields() {
11126        let registry = Arc::new(Registry::default());
11127        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
11128            [
11129                ("consumer".to_string(), true),
11130                ("provider".to_string(), false),
11131            ],
11132            BTreeMap::new(),
11133        );
11134        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11135        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11136            provides: Vec::new(),
11137            requires: vec![subc_protocol::manifest::CapabilityRequirement {
11138                capability: "credentials-provider/v1".to_string(),
11139                need: subc_protocol::manifest::CapabilityNeed::Required,
11140            }],
11141            must_never_reach: Vec::new(),
11142        });
11143        let hello = Frame::build(
11144            FrameType::Hello,
11145            control_flags(),
11146            0,
11147            0,
11148            78,
11149            serde_json::to_vec(&ModuleHelloBody {
11150                manifest: consumer,
11151                protocol_ver: PROTOCOL_VERSION,
11152                control_ops: None,
11153                launch_nonce: None,
11154            })
11155            .expect("HELLO serializes"),
11156        )
11157        .expect("HELLO frame builds");
11158        handler
11159            .handle_control(ConnectionId::new(78), hello)
11160            .expect("consumer registers");
11161        let describe = Frame::build(
11162            FrameType::Request,
11163            control_flags(),
11164            0,
11165            0,
11166            79,
11167            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11168                .expect("request serializes"),
11169        )
11170        .expect("describe frame builds");
11171        let response = handler
11172            .handle_server_describe(describe)
11173            .expect("server.describe succeeds");
11174        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11175        let requirement = &rendered["capability_requirements"][0];
11176        assert_eq!(requirement["consumer"], "consumer");
11177        assert_eq!(requirement["verdict"], "never_provided");
11178        assert_eq!(requirement["episode_seq"], 1);
11179        assert_eq!(requirement["config_satisfiable"], false);
11180        assert_eq!(requirement["runtime_available"], false);
11181        assert!(requirement["detail"]
11182            .as_str()
11183            .expect("detail string")
11184            .contains("credentials-provider/v1"));
11185    }
11186
11187    /// Register a consumer that requires `project-identity/v1` and the provider
11188    /// of it, optionally mark daemon shutdown as begun, then deregister the
11189    /// provider as its connection closes. Returns the `capability.requirement`
11190    /// events that deregistration logged and the verdict the evaluator then
11191    /// holds for the requirement (the one `route.open` refusals read).
11192    fn deregister_required_provider(
11193        daemon_shutdown_begun: bool,
11194    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11195        const CAPABILITY: &str = "project-identity/v1";
11196        let registry = Arc::new(Registry::default());
11197        let daemon_shutdown = DaemonShutdownFlag::default();
11198        let handler = ControlHandler::new(Arc::clone(&registry))
11199            .with_capability_config(
11200                [
11201                    ("consumer".to_string(), true),
11202                    ("provider".to_string(), true),
11203                ],
11204                BTreeMap::new(),
11205            )
11206            .with_daemon_shutdown_flag(daemon_shutdown.clone());
11207        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11208        consumer.capabilities = Some(CapabilityDeclarations {
11209            provides: Vec::new(),
11210            requires: vec![subc_protocol::manifest::CapabilityRequirement {
11211                capability: CAPABILITY.to_string(),
11212                need: subc_protocol::manifest::CapabilityNeed::Required,
11213            }],
11214            must_never_reach: Vec::new(),
11215        });
11216        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11217        let provider_conn = ConnectionId::new(182);
11218        for (conn, manifest, corr) in [
11219            (ConnectionId::new(181), consumer, 181),
11220            (provider_conn, provider, 182),
11221        ] {
11222            let hello = handler
11223                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11224                .expect("module registers");
11225            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11226        }
11227        assert_eq!(
11228            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11229            Some(CapabilityVerdict::Provided),
11230            "the registered provider must satisfy the requirement first"
11231        );
11232        if daemon_shutdown_begun {
11233            daemon_shutdown.set_for_test();
11234        }
11235
11236        let capture = EventCapture::default();
11237        let _subscriber =
11238            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11239        handler
11240            .cleanup_connection(provider_conn)
11241            .expect("provider deregisters");
11242        let events = capture
11243            .events()
11244            .into_iter()
11245            .filter(|event| {
11246                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11247            })
11248            .collect();
11249        (
11250            events,
11251            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11252        )
11253    }
11254
11255    #[test]
11256    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11257        let (events, verdict) = deregister_required_provider(true);
11258        assert_eq!(
11259            verdict,
11260            Some(CapabilityVerdict::NeverProvided),
11261            "shutdown must not change the verdict route.open refusals read"
11262        );
11263        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11264        let event = &events[0];
11265        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11266        assert_eq!(
11267            event.fields.get("cause").map(String::as_str),
11268            Some("\"daemon_shutdown\""),
11269            "{event:?}"
11270        );
11271        assert_eq!(
11272            event.fields.get("verdict").map(String::as_str),
11273            Some("\"never_provided\""),
11274            "{event:?}"
11275        );
11276    }
11277
11278    #[test]
11279    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11280        let (events, verdict) = deregister_required_provider(false);
11281        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11282        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11283        let event = &events[0];
11284        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11285        assert_eq!(event.fields.get("cause"), None, "{event:?}");
11286        assert_eq!(
11287            event.fields.get("verdict").map(String::as_str),
11288            Some("\"never_provided\""),
11289            "{event:?}"
11290        );
11291    }
11292
11293    #[test]
11294    fn catalog_list_omits_capabilities_for_legacy_manifest() {
11295        let registry = Arc::new(Registry::default());
11296        let handler = ControlHandler::new(Arc::clone(&registry));
11297        let hello = handler
11298            .handle_control(
11299                ConnectionId::new(101),
11300                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11301            )
11302            .expect("legacy HELLO registers");
11303        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11304
11305        let request = Frame::build(
11306            FrameType::Request,
11307            control_flags(),
11308            0,
11309            0,
11310            102,
11311            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11312                .expect("catalog request serializes"),
11313        )
11314        .expect("catalog request frame builds");
11315        let response = handler
11316            .handle_catalog_list(request, None)
11317            .expect("catalog list succeeds");
11318        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11319        assert!(
11320            body["modules"][0].get("capabilities").is_none(),
11321            "legacy manifest must retain an absent capabilities field on catalog.list"
11322        );
11323    }
11324
11325    #[test]
11326    fn hello_ack_omits_storage_when_no_storage_config() {
11327        let registry = Arc::new(Registry::default());
11328        let handler = ControlHandler::new(Arc::clone(&registry));
11329        let responses = handler
11330            .handle_control(
11331                ConnectionId::new(1),
11332                hello_frame("aft", PROTOCOL_VERSION, 7),
11333            )
11334            .unwrap();
11335        let ack = parse_ack(&responses[0]);
11336        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11337        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11338    }
11339
11340    #[tokio::test]
11341    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11342        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11343        let registry = Arc::new(Registry::default());
11344        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
11345        let responses = handler
11346            .handle_control(
11347                ConnectionId::new(1),
11348                hello_frame("aft", PROTOCOL_VERSION, 7),
11349            )
11350            .unwrap();
11351        let ack = parse_ack(&responses[0]);
11352        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11353
11354        let described = handler
11355            .handle_control_frame(
11356                &route_ctx(ConnectionId::new(2)).0,
11357                Frame::build(
11358                    FrameType::Request,
11359                    control_flags(),
11360                    0,
11361                    0,
11362                    9,
11363                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11364                )
11365                .unwrap(),
11366            )
11367            .await
11368            .unwrap();
11369        let ClientControlResponse::ServerDescribe { machine_id, .. } =
11370            serde_json::from_slice(&described[0].body).unwrap()
11371        else {
11372            panic!("server.describe answered with another shape");
11373        };
11374        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11375    }
11376
11377    #[test]
11378    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11379        // With a central sqlite storage policy, each registering module gets its
11380        // own resolved descriptor in HELLO_ACK, keyed by its module id.
11381        let registry = Arc::new(Registry::default());
11382        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
11383            crate::daemon_config::StorageConfig::Sqlite {
11384                data_home: std::path::PathBuf::from("/data"),
11385            },
11386        ));
11387
11388        let responses = handler
11389            .handle_control(
11390                ConnectionId::new(1),
11391                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11392            )
11393            .unwrap();
11394        let ack = parse_ack(&responses[0]);
11395        assert_eq!(
11396            ack.storage,
11397            Some(serde_json::json!({
11398                "module_id": "alfonso-routing",
11399                "storage_namespace": "default",
11400                "isolation": { "kind": "module" },
11401                "backend": {
11402                    "backend": "sqlite",
11403                    "path": "/data/cortexkit/alfonso-routing/store.db"
11404                }
11405            })),
11406            "the delivered descriptor is the module's own sqlite store path"
11407        );
11408    }
11409
11410    #[test]
11411    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11412        let registry = Arc::new(Registry::default());
11413        let handler = ControlHandler::new(Arc::clone(&registry));
11414        let conn = ConnectionId::new(1);
11415        let responses = handler
11416            .handle_control(
11417                conn,
11418                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11419            )
11420            .unwrap();
11421        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11422        let registration = registry.get_module("aft").unwrap().unwrap();
11423        assert_eq!(registration.control_ops, module_baseline_control_ops());
11424
11425        let frame =
11426            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11427        assert!(handler
11428            .guard_module_control_op(&frame, "aft", "route.bind")
11429            .unwrap()
11430            .is_none());
11431        let error = handler
11432            .guard_module_control_op(&frame, "aft", "test.synthetic")
11433            .unwrap()
11434            .expect("synthetic ungranted op should be rejected");
11435        assert_eq!(error.header.ty, FrameType::Error);
11436        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11437    }
11438
11439    #[test]
11440    fn hello_control_ops_some_adds_optional_grants() {
11441        let registry = Arc::new(Registry::default());
11442        let handler = ControlHandler::new(Arc::clone(&registry));
11443        handler
11444            .handle_control(
11445                ConnectionId::new(1),
11446                hello_frame_with_control_ops(
11447                    "aft",
11448                    PROTOCOL_VERSION,
11449                    7,
11450                    Some(vec![
11451                        "future.synthetic".to_string(),
11452                        "route.bind".to_string(),
11453                    ]),
11454                ),
11455            )
11456            .unwrap();
11457        let registration = registry.get_module("aft").unwrap().unwrap();
11458        assert_eq!(
11459            registration.control_ops,
11460            vec![
11461                "route.bind".to_string(),
11462                "route.status".to_string(),
11463                "future.synthetic".to_string(),
11464            ]
11465        );
11466        let frame =
11467            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11468        assert!(handler
11469            .guard_module_control_op(&frame, "aft", "future.synthetic")
11470            .unwrap()
11471            .is_none());
11472    }
11473
11474    #[tokio::test]
11475    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11476        let registry = Arc::new(Registry::default());
11477        let forwarding = Arc::new(ForwardingTable::default());
11478        let handler =
11479            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11480        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11481        hello_via_sink(
11482            &handler,
11483            &module_ctx,
11484            &mut module_rx,
11485            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11486        )
11487        .await;
11488
11489        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11490        let responses = handler
11491            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11492            .await
11493            .unwrap();
11494        assert_eq!(responses.len(), 1);
11495        assert_eq!(responses[0].header.ty, FrameType::Error);
11496        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11497        assert!(module_rx.try_recv().is_err());
11498    }
11499
11500    #[tokio::test]
11501    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11502        let registry = Arc::new(Registry::default());
11503        let forwarding = Arc::new(ForwardingTable::default());
11504        let handler =
11505            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11506        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11507        hello_via_sink(
11508            &handler,
11509            &module_ctx,
11510            &mut module_rx,
11511            hello_frame_with_control_ops(
11512                "aft",
11513                PROTOCOL_VERSION,
11514                7,
11515                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11516            ),
11517        )
11518        .await;
11519
11520        let project_root = unique_project_root("demux");
11521        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11522        let route_handler = handler.clone();
11523        let route_task = tokio::spawn(async move {
11524            route_handler
11525                .handle_control_frame(
11526                    &route_client_ctx,
11527                    route_open_frame(100, "aft", project_root),
11528                )
11529                .await
11530                .unwrap()
11531        });
11532        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11533            .await
11534            .unwrap()
11535            .unwrap();
11536        assert!(matches!(
11537            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11538            ModuleControlRequest::RouteBind { .. }
11539        ));
11540
11541        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11542        let health_handler = handler.clone();
11543        let health_task = tokio::spawn(async move {
11544            health_handler
11545                .handle_control_frame(
11546                    &health_client_ctx,
11547                    supervisor_health_probe_frame(101, "aft"),
11548                )
11549                .await
11550                .unwrap()
11551        });
11552        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11553            .await
11554            .unwrap()
11555            .unwrap();
11556        assert_eq!(
11557            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11558            ModuleControlRequest::HealthCheck {}
11559        );
11560
11561        handler
11562            .handle_control_frame(
11563                &module_ctx,
11564                health_response(health_frame.header.corr, HealthStatus::Degraded),
11565            )
11566            .await
11567            .unwrap();
11568        let health_response = health_task.await.unwrap();
11569        assert_eq!(health_response.len(), 1);
11570        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11571            ClientControlResponse::SupervisorHealthProbe {
11572                module_id,
11573                status,
11574                detail,
11575                metrics,
11576            } => {
11577                assert_eq!(module_id, "aft");
11578                assert_eq!(status, HealthStatus::Degraded);
11579                assert_eq!(detail.as_deref(), Some("warming"));
11580                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11581            }
11582            other => panic!("unexpected health response: {other:?}"),
11583        }
11584
11585        handler
11586            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11587            .await
11588            .unwrap();
11589        let route_response = route_task.await.unwrap();
11590        assert!(route_response.is_empty());
11591        let published = route_client_rx.recv().await.unwrap();
11592        assert!(matches!(
11593            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11594            ClientControlResponse::RouteOpen { .. }
11595        ));
11596    }
11597
11598    /// Start one `route.open` on `client_connection` and return its still-running
11599    /// handler task together with the `route.bind` the module received for it.
11600    /// The handler blocks until the module answers, so it has to run as a task
11601    /// while the test drives the module side.
11602    async fn relay_route_open(
11603        handler: &ControlHandler,
11604        client_connection: ConnectionId,
11605        client_egress: &FrameSink,
11606        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11607        corr: u64,
11608        module_id: &str,
11609        project_root_label: &str,
11610    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11611        let ctx = RouteCtx {
11612            connection_id: client_connection,
11613            egress: client_egress.clone(),
11614        };
11615        let handler = handler.clone();
11616        let project_root = unique_project_root(project_root_label);
11617        let module_id = module_id.to_string();
11618        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11619        let task = tokio::spawn(async move {
11620            let _guard = tracing::dispatcher::set_default(&dispatch);
11621            handler
11622                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11623                .await
11624                .unwrap()
11625        });
11626        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11627            .await
11628            .expect("module receives the relayed route.bind")
11629            .expect("module egress is open");
11630        (task, bind.frame)
11631    }
11632
11633    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11634        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11635            ModuleControlRequest::RouteBind {
11636                route_channel,
11637                epoch,
11638                ..
11639            } => (route_channel, epoch),
11640            other => panic!("expected a route.bind request, got {other:?}"),
11641        }
11642    }
11643
11644    fn published_route(frame: &Frame) -> (u16, u32) {
11645        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11646            ClientControlResponse::RouteOpen {
11647                route_channel,
11648                route_epoch,
11649                ..
11650            } => (route_channel, route_epoch),
11651            other => panic!("expected a route.open response, got {other:?}"),
11652        }
11653    }
11654
11655    /// Reproduction of a production outage. A client had `route.open`s in
11656    /// flight to a module and was already marked closing -- its egress had refused a
11657    /// module frame, so the daemon asked its connection to end -- while its sink
11658    /// was still open. When the module acked those binds, the daemon refused to
11659    /// commit a route for a closing client, and that refusal was returned from
11660    /// the MODULE connection's frame handler, where a router error that has no
11661    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
11662    /// the supervisor correctly did not respawn a clean exit, and every seat lost
11663    /// its tools for hours -- one client's teardown took down a connection
11664    /// carrying ~170 other routes.
11665    ///
11666    /// The window is opened here by calling the production path that opens it
11667    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
11668    /// state that matters is "in `closing_connections`, sink still open, relay
11669    /// still pending", and it lasts only from the close request until the
11670    /// connection loop reacts to it; a socket-level test can flood a client into
11671    /// that escalation but cannot pin the module's ack inside the window. Closing
11672    /// the socket instead takes the other path entirely -- connection teardown
11673    /// removes the pending relay under the same lock, so the ack finds nothing.
11674    #[tokio::test]
11675    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11676        let registry = Arc::new(Registry::default());
11677        let forwarding = Arc::new(ForwardingTable::default());
11678        let handler =
11679            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11680
11681        let module_connection = ConnectionId::new(30);
11682        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11683        hello_via_sink(
11684            &handler,
11685            &module_ctx,
11686            &mut module_rx,
11687            hello_frame("aft", PROTOCOL_VERSION, 7),
11688        )
11689        .await;
11690
11691        let dying_client = ConnectionId::new(31);
11692        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11693
11694        // A published route on the dying client. The escalation below only marks
11695        // a connection closing for a route it has already published.
11696        let (first_task, first_bind) = relay_route_open(
11697            &handler,
11698            dying_client,
11699            &dying_ctx.egress,
11700            &mut module_rx,
11701            100,
11702            "aft",
11703            "closing-first",
11704        )
11705        .await;
11706        handler
11707            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11708            .await
11709            .unwrap();
11710        assert!(first_task.await.unwrap().is_empty());
11711        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11712
11713        // A second route.open from the same client, relayed and awaiting its ack.
11714        let (second_task, second_bind) = relay_route_open(
11715            &handler,
11716            dying_client,
11717            &dying_ctx.egress,
11718            &mut module_rx,
11719            101,
11720            "aft",
11721            "closing-second",
11722        )
11723        .await;
11724        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11725
11726        // The window: the client is closing, its sink is still open, and its
11727        // second bind is still pending.
11728        assert!(forwarding
11729            .escalate_client_delivery_failure(
11730                dying_client,
11731                first_channel,
11732                first_epoch,
11733                CloseReason::new(
11734                    "module_to_client_delivery_failed",
11735                    "client egress refused a module frame",
11736                ),
11737                crate::forwarding::UndeliveredFrame {
11738                    module_id: None,
11739                    sink: &dying_ctx.egress,
11740                },
11741            )
11742            .unwrap());
11743        assert!(!dying_ctx.egress.is_closed());
11744
11745        // The frame that used to end the module connection.
11746        let ack = handler
11747            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11748            .await;
11749        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11750        if module_loop_error.is_some() {
11751            // What the server's connection loop does with a router error that has
11752            // no ERROR-frame translation: end the connection, which releases the
11753            // module's registration and every route on it.
11754            handler.cleanup_connection(module_connection).unwrap();
11755        }
11756        // Read the module's next frame before opening the co-tenant's route, so
11757        // the GOODBYE assertion below is about THIS ack and not about later
11758        // traffic. `None` means the module was told nothing.
11759        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11760            .await
11761            .ok()
11762            .flatten();
11763
11764        // 1. The module connection is still registered.
11765        assert!(
11766            registry
11767                .get_module_by_connection(module_connection)
11768                .unwrap()
11769                .is_some(),
11770            "one client's closing connection ended the shared module connection: \
11771             {module_loop_error:?}"
11772        );
11773        // ...and still serving: another client can open and use a route on it.
11774        let cotenant = ConnectionId::new(32);
11775        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11776        let (cotenant_task, cotenant_bind) = relay_route_open(
11777            &handler,
11778            cotenant,
11779            &cotenant_ctx.egress,
11780            &mut module_rx,
11781            102,
11782            "aft",
11783            "closing-cotenant",
11784        )
11785        .await;
11786        handler
11787            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11788            .await
11789            .unwrap();
11790        assert!(cotenant_task.await.unwrap().is_empty());
11791        let (cotenant_channel, cotenant_epoch) =
11792            published_route(&cotenant_rx.recv().await.unwrap());
11793        assert!(matches!(
11794            forwarding
11795                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11796                .unwrap(),
11797            DataRoute::Client(DataRouteState::Bound(_))
11798        ));
11799
11800        // 2. The module was told to drop the binding it created for the route
11801        //    that will never be published.
11802        let goodbye = post_ack_module_frame
11803            .expect("module receives a GOODBYE for the abandoned route channel");
11804        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11805        assert_eq!(goodbye.header.channel, abandoned_channel);
11806        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11807
11808        // 3. The dying client received nothing: no route was ever published to
11809        //    it. Its route.open is answered as unavailable, which the connection
11810        //    loop would write to a socket that is already going away.
11811        assert!(dying_rx.try_recv().is_err());
11812        let second_response = second_task.await.unwrap();
11813        assert_eq!(second_response.len(), 1);
11814        assert_eq!(
11815            parse_error(&second_response[0])["code"],
11816            "target_unavailable"
11817        );
11818    }
11819
11820    /// One `catalog.list` as a client sees it: the generation and the listed
11821    /// module ids with their readiness.
11822    async fn catalog_snapshot(
11823        handler: &ControlHandler,
11824        client_ctx: &RouteCtx,
11825        corr: u64,
11826    ) -> (u64, Vec<(String, bool)>) {
11827        let body =
11828            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11829        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11830        let reply = handler
11831            .handle_control_frame(client_ctx, request)
11832            .await
11833            .unwrap()
11834            .pop()
11835            .unwrap();
11836        let ClientControlResponse::CatalogList {
11837            generation,
11838            modules,
11839            ..
11840        } = serde_json::from_slice(&reply.body).unwrap()
11841        else {
11842            panic!("not a catalog.list reply");
11843        };
11844        (
11845            generation,
11846            modules
11847                .into_iter()
11848                .map(|entry| (entry.module_id, entry.ready))
11849                .collect(),
11850        )
11851    }
11852
11853    /// A handler whose HELLO for module "aft" on connection 40 is parked
11854    /// between reserving its registry entry and installing its forwarding
11855    /// endpoint. Returns the handler, the two-party barrier (wait once to know
11856    /// the HELLO is parked, once more to release it), the HELLO's thread, and
11857    /// the module's context and receiver.
11858    fn hello_held_before_forwarding() -> (
11859        ControlHandler,
11860        Arc<std::sync::Barrier>,
11861        std::thread::JoinHandle<Vec<Frame>>,
11862        RouteCtx,
11863        mpsc::Receiver<crate::router::OutboundFrame>,
11864    ) {
11865        let hold = Arc::new(std::sync::Barrier::new(2));
11866        let handler = ControlHandler::with_forwarding(
11867            Arc::new(Registry::default()),
11868            Arc::new(ForwardingTable::default()),
11869        )
11870        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11871        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11872        let hello = {
11873            let handler = handler.clone();
11874            let module_ctx = module_ctx.clone();
11875            std::thread::spawn(move || {
11876                handler
11877                    .handle_hello(
11878                        module_ctx.connection_id,
11879                        Some(module_ctx.egress.clone()),
11880                        hello_frame("aft", PROTOCOL_VERSION, 7),
11881                    )
11882                    .unwrap()
11883            })
11884        };
11885        (handler, hold, hello, module_ctx, module_rx)
11886    }
11887
11888    /// While a HELLO has reserved its registry entry but its forwarding
11889    /// endpoint is not installed yet, `route.open` to the module can only be
11890    /// refused. `catalog.list` must not list the module in that window: a
11891    /// client that opens a route as soon as it sees a module listed and ready
11892    /// would be refused for a module the daemon had just advertised. The
11893    /// refusal in the window stays the retryable `target_unavailable`, never
11894    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
11895    /// Once the HELLO completes, the module is listed and a route to it opens.
11896    #[tokio::test]
11897    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11898        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11899        let client = ConnectionId::new(41);
11900        let (client_ctx, mut client_rx) = route_ctx(client);
11901
11902        hold.wait();
11903        // Parked: the registry holds "aft", forwarding has no endpoint for it.
11904        assert!(handler.registry.get_module("aft").unwrap().is_some());
11905        assert!(!handler
11906            .forwarding
11907            .has_live_module_connection("aft")
11908            .unwrap());
11909        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11910        assert!(
11911            listed.is_empty(),
11912            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11913        );
11914        let refused = handler
11915            .handle_control_frame(
11916                &client_ctx,
11917                route_open_frame(301, "aft", unique_project_root("listed-window")),
11918            )
11919            .await
11920            .unwrap();
11921        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11922        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11923        let duplicate = handler
11924            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11925            .await
11926            .unwrap();
11927        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11928
11929        hold.wait();
11930        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11931        let ack = module_rx.try_recv().unwrap().frame;
11932        assert_eq!(ack.header.ty, FrameType::HelloAck);
11933        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11934        assert_eq!(listed, vec![("aft".to_string(), true)]);
11935
11936        let (open_task, bind) = relay_route_open(
11937            &handler,
11938            client,
11939            &client_ctx.egress,
11940            &mut module_rx,
11941            303,
11942            "aft",
11943            "listed-after",
11944        )
11945        .await;
11946        assert!(handler
11947            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11948            .await
11949            .unwrap()
11950            .is_empty());
11951        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11952        published_route(&client_rx.try_recv().unwrap().frame);
11953    }
11954
11955    /// A catalog watcher keys on the generation: when it changes, it re-reads
11956    /// the listing. So the generation must advance exactly when the module
11957    /// appears. If it advanced while the module was still hidden, a watcher
11958    /// would read the new generation without the module, and the module would
11959    /// later appear under that same generation with nothing telling the
11960    /// watcher to look again.
11961    #[tokio::test]
11962    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11963        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11964        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11965        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11966
11967        hold.wait();
11968        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11969        assert!(listed.is_empty());
11970        assert_eq!(
11971            during, before,
11972            "the generation moved while the module was not listed"
11973        );
11974
11975        hold.wait();
11976        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11977        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11978        assert_eq!(listed, vec![("aft".to_string(), true)]);
11979        assert_eq!(
11980            after,
11981            before + 1,
11982            "the module appeared without exactly one generation bump"
11983        );
11984    }
11985
11986    /// A HELLO whose forwarding endpoint cannot be installed (here the
11987    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
11988    /// refused and leaves nothing behind: no registry entry, no catalog entry,
11989    /// and the catalog generation where it was.
11990    #[tokio::test]
11991    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11992        let registry = Arc::new(Registry::default());
11993        let handler = ControlHandler::with_forwarding(
11994            Arc::clone(&registry),
11995            Arc::new(ForwardingTable::default()),
11996        );
11997        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11998        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11999        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12000        drop(module_rx);
12001
12002        let replies = handler
12003            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12004            .await
12005            .unwrap();
12006        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12007        assert!(registry.get_module("aft").unwrap().is_none());
12008        assert!(registry
12009            .get_module_by_connection(module_ctx.connection_id)
12010            .unwrap()
12011            .is_none());
12012        assert_eq!(
12013            catalog_snapshot(&handler, &client_ctx, 301).await,
12014            (before, Vec::new())
12015        );
12016    }
12017
12018    /// A handler whose `route.open` parks after queueing the route.bind and
12019    /// before marking it enqueued, with module "aft" registered on connection
12020    /// 40. Returns the two-party barrier that parks and releases it.
12021    async fn route_open_held_after_bind_send() -> (
12022        ControlHandler,
12023        Arc<tokio::sync::Barrier>,
12024        RouteCtx,
12025        mpsc::Receiver<crate::router::OutboundFrame>,
12026    ) {
12027        let hold = Arc::new(tokio::sync::Barrier::new(2));
12028        let handler = ControlHandler::with_forwarding(
12029            Arc::new(Registry::default()),
12030            Arc::new(ForwardingTable::default()),
12031        )
12032        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12033        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12034        hello_via_sink(
12035            &handler,
12036            &module_ctx,
12037            &mut module_rx,
12038            hello_frame("aft", PROTOCOL_VERSION, 7),
12039        )
12040        .await;
12041        (handler, hold, module_ctx, module_rx)
12042    }
12043
12044    /// A fast module acks the route.bind, and the reader commits the route,
12045    /// before `route.open` has marked the relay enqueued. The daemon used to
12046    /// read the missing relay as an abandoned bind and send the module a
12047    /// GOODBYE for the route it had just committed: the client was told its
12048    /// route was open, the module unbound the channel, and the client's first
12049    /// request was dropped on the module side as unbound.
12050    #[tokio::test]
12051    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12052        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12053        let client = ConnectionId::new(41);
12054        let (client_ctx, mut client_rx) = route_ctx(client);
12055
12056        let (open_task, bind) = relay_route_open(
12057            &handler,
12058            client,
12059            &client_ctx.egress,
12060            &mut module_rx,
12061            200,
12062            "aft",
12063            "acked-before-mark",
12064        )
12065        .await;
12066        let (module_channel, module_epoch) = route_bind_channel(&bind);
12067        // route.open is now parked between queueing the bind and marking it.
12068        hold.wait().await;
12069
12070        // The module answers inside that window, and the route commits.
12071        assert!(handler
12072            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12073            .await
12074            .unwrap()
12075            .is_empty());
12076        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12077
12078        hold.wait().await;
12079        assert!(
12080            open_task.await.unwrap().is_empty(),
12081            "route.open answers accepted"
12082        );
12083
12084        // Every frame route.open could have sent is already queued, because it
12085        // has returned; the module must have been sent nothing.
12086        match module_rx.try_recv() {
12087            Err(mpsc::error::TryRecvError::Empty) => {}
12088            Ok(outbound) => panic!(
12089                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12090                outbound.frame.header.ty,
12091                outbound.frame.header.channel,
12092                outbound.frame.header.epoch
12093            ),
12094            Err(err) => panic!("module egress closed: {err}"),
12095        }
12096
12097        // The client's first request on the route is the next thing the module
12098        // reads, on the binding it just created.
12099        let router = Router::with_control_handler(Arc::new(handler.clone()));
12100        let request = Frame::build(
12101            FrameType::Request,
12102            Flags::new(true, Priority::Interactive, false),
12103            client_channel,
12104            client_epoch,
12105            201,
12106            b"first request".to_vec(),
12107        )
12108        .unwrap();
12109        router
12110            .route_for_connection(&client_ctx, request)
12111            .await
12112            .unwrap();
12113        let delivered = module_rx.try_recv().unwrap().frame;
12114        assert_eq!(delivered.header.ty, FrameType::Request);
12115        assert_eq!(
12116            (delivered.header.channel, delivered.header.epoch),
12117            (module_channel, module_epoch)
12118        );
12119        assert_eq!(delivered.body, b"first request");
12120    }
12121
12122    /// The client goes away while `route.open` is still queueing the bind, so
12123    /// the relay is dropped with no module reply. The teardown cannot send the
12124    /// GOODBYE itself (it could reach the module ahead of the bind), so
12125    /// `route.open` must send it once the bind is queued; otherwise the module
12126    /// keeps a binding the daemon will never commit.
12127    #[tokio::test]
12128    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12129        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12130        let client = ConnectionId::new(42);
12131        let (client_ctx, _client_rx) = route_ctx(client);
12132
12133        let (open_task, bind) = relay_route_open(
12134            &handler,
12135            client,
12136            &client_ctx.egress,
12137            &mut module_rx,
12138            300,
12139            "aft",
12140            "released-before-mark",
12141        )
12142        .await;
12143        let (module_channel, module_epoch) = route_bind_channel(&bind);
12144        hold.wait().await;
12145
12146        handler.cleanup_connection(client).unwrap();
12147        assert!(
12148            module_rx.try_recv().is_err(),
12149            "client teardown must not GOODBYE a bind route.open is still queueing"
12150        );
12151
12152        hold.wait().await;
12153        let response = open_task.await.unwrap();
12154        assert_eq!(response.len(), 1);
12155        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12156
12157        let goodbye = module_rx
12158            .try_recv()
12159            .expect("module is told to drop the binding the daemon abandoned")
12160            .frame;
12161        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12162        assert_eq!(
12163            (goodbye.header.channel, goodbye.header.epoch),
12164            (module_channel, module_epoch)
12165        );
12166        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12167    }
12168
12169    /// The module acks inside the same window, but the client's egress closed
12170    /// first, so the reader refuses to commit. The ack proves the module holds
12171    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
12172    /// send a second one.
12173    #[tokio::test]
12174    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12175        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12176        let client = ConnectionId::new(43);
12177        let (client_ctx, client_rx) = route_ctx(client);
12178
12179        let (open_task, bind) = relay_route_open(
12180            &handler,
12181            client,
12182            &client_ctx.egress,
12183            &mut module_rx,
12184            400,
12185            "aft",
12186            "gone-client-before-mark",
12187        )
12188        .await;
12189        let (module_channel, module_epoch) = route_bind_channel(&bind);
12190        hold.wait().await;
12191
12192        drop(client_rx);
12193        assert!(client_ctx.egress.is_closed());
12194        handler
12195            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12196            .await
12197            .unwrap();
12198
12199        hold.wait().await;
12200        let response = open_task.await.unwrap();
12201        assert_eq!(response.len(), 1);
12202        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12203
12204        let goodbye = module_rx
12205            .try_recv()
12206            .expect("module is told to drop the binding it acked")
12207            .frame;
12208        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12209        assert_eq!(
12210            (goodbye.header.channel, goodbye.header.epoch),
12211            (module_channel, module_epoch)
12212        );
12213        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12214        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12215    }
12216
12217    /// The fence at the module-loop boundary, stated as its own contract: which
12218    /// forwarding failures are allowed to end the module connection that is being
12219    /// served. A `ConnectionClosing` naming some client is about that client, and
12220    /// a module connection is shared; the same error naming the module's own
12221    /// connection is about this connection and must stay fatal, as must failures
12222    /// that are about the forwarding table itself.
12223    #[test]
12224    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12225        let handler = ControlHandler::default();
12226        let module_connection = ConnectionId::new(30);
12227        let client_connection = ConnectionId::new(31);
12228
12229        handler
12230            .refuse_to_end_module_connection_for_a_client(
12231                module_connection,
12232                77,
12233                ForwardingError::ConnectionClosing {
12234                    connection_id: client_connection,
12235                },
12236            )
12237            .expect("a closing client must never end the module connection");
12238
12239        assert!(matches!(
12240            handler.refuse_to_end_module_connection_for_a_client(
12241                module_connection,
12242                78,
12243                ForwardingError::ConnectionClosing {
12244                    connection_id: module_connection,
12245                },
12246            ),
12247            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12248                connection_id
12249            })) if connection_id == module_connection
12250        ));
12251        assert!(matches!(
12252            handler.refuse_to_end_module_connection_for_a_client(
12253                module_connection,
12254                79,
12255                ForwardingError::Poisoned,
12256            ),
12257            Err(RouterError::Forwarding(ForwardingError::Poisoned))
12258        ));
12259        assert!(matches!(
12260            handler.refuse_to_end_module_connection_for_a_client(
12261                module_connection,
12262                80,
12263                ForwardingError::StaleModuleEndpoint,
12264            ),
12265            Err(RouterError::Forwarding(
12266                ForwardingError::StaleModuleEndpoint
12267            ))
12268        ));
12269    }
12270
12271    /// The spawn-attestation guard is what stops a connected module from claiming
12272    /// another module's identity and being stamped `Reserved` for it. Every other
12273    /// test that supplies a consumer_identity supplies a CORRECT one, because a
12274    /// correct one is what the rest of the flow needs -- so the guard's rejection
12275    /// branch was never the subject of an assertion, only its acceptance branch.
12276    ///
12277    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
12278    /// whole subc-core library suite green; only the forwarding integration tests
12279    /// notice, and they notice for unrelated reasons. This test exists so the
12280    /// refusal itself is asserted where the guard lives: it fails if the identity
12281    /// check stops refusing, which is the direction that matters, since a guard
12282    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
12283    #[tokio::test]
12284    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12285        let registry = Arc::new(Registry::default());
12286        let forwarding = Arc::new(ForwardingTable::default());
12287        let supervisor = SupervisorHandle::new();
12288        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12289        let handler =
12290            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12291                .with_supervisor(supervisor);
12292
12293        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12294        hello_via_sink(
12295            &handler,
12296            &target_ctx,
12297            &mut target_rx,
12298            hello_frame("target", PROTOCOL_VERSION, 1),
12299        )
12300        .await;
12301
12302        // A real supervised module id presenting the wrong nonce. This is the
12303        // impersonation case: the attacker knows a privileged module_id, which is
12304        // public, and guesses at the nonce, which is not.
12305        let wrong_nonce = handler
12306            .handle_control_frame(
12307                &route_ctx(ConnectionId::new(91)).0,
12308                route_open_frame_with_admission_facts(
12309                    20,
12310                    "target",
12311                    unique_project_root("admission-facts"),
12312                    Some(subc_control::ConsumerIdentity {
12313                        module_id: "fed".to_string(),
12314                        launch_nonce: "not-the-real-nonce".to_string(),
12315                    }),
12316                    None,
12317                ),
12318            )
12319            .await
12320            .unwrap();
12321        assert_eq!(
12322            parse_error(&wrong_nonce[0])["code"],
12323            "bad_consumer_identity",
12324            "a mismatched launch nonce must be refused, not stamped Reserved"
12325        );
12326
12327        // A module id the supervisor never spawned at all, so no nonce exists to
12328        // compare against. An implementation that treats "no record" as "nothing
12329        // to check" fails open here while passing the case above.
12330        let never_spawned = handler
12331            .handle_control_frame(
12332                &route_ctx(ConnectionId::new(92)).0,
12333                route_open_frame_with_admission_facts(
12334                    21,
12335                    "target",
12336                    unique_project_root("admission-facts"),
12337                    Some(subc_control::ConsumerIdentity {
12338                        module_id: "never-spawned".to_string(),
12339                        launch_nonce: "any-nonce".to_string(),
12340                    }),
12341                    None,
12342                ),
12343            )
12344            .await
12345            .unwrap();
12346        assert_eq!(
12347            parse_error(&never_spawned[0])["code"],
12348            "bad_consumer_identity",
12349            "an unspawned module_id must be refused rather than accepted for lack of a record"
12350        );
12351    }
12352
12353    /// The refusal test above proves the guard says NO. Nothing proved it can say
12354    /// YES, and the difference is not academic: replacing the whole authorization
12355    /// with `false` -- admitting no consumer identity at all, revoking Reserved
12356    /// standing for every supervised module in the fleet -- leaves 110 of the 111
12357    /// library tests GREEN. The one that notices does so by HANGING, because it
12358    /// waits for a bind that can no longer happen.
12359    ///
12360    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
12361    /// or flaky test, invites a RETRY rather than an investigation, and the retry
12362    /// hangs too and gets blamed on the runner. So a total revocation of the
12363    /// daemon's trust grant would have shipped behind a symptom nobody attributes
12364    /// to code.
12365    ///
12366    /// The bias is structural rather than accidental. A REFUSAL looks like a
12367    /// failure someone writes a test for; a GRANT looks like the happy path. Every
12368    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
12369    /// suite for that reason, and this one is the purest case in the daemon.
12370    ///
12371    /// This test asserts the EFFECT rather than the absence of an error: the module
12372    /// receives a RouteBind and it carries `Reserved` naming the attested module.
12373    /// A guard that admitted nobody would produce no bind at all; one that admitted
12374    /// everybody would stamp the wrong principal, which the refusal test catches.
12375    #[tokio::test]
12376    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12377        let registry = Arc::new(Registry::default());
12378        let forwarding = Arc::new(ForwardingTable::default());
12379        let supervisor = SupervisorHandle::new();
12380        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12381        let handler =
12382            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12383                .with_supervisor(supervisor);
12384
12385        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12386        hello_via_sink(
12387            &handler,
12388            &target_ctx,
12389            &mut target_rx,
12390            hello_frame("target", PROTOCOL_VERSION, 1),
12391        )
12392        .await;
12393
12394        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12395        let route_handler = handler.clone();
12396        let route_task = tokio::spawn(async move {
12397            route_handler
12398                .handle_control_frame(
12399                    &client_ctx,
12400                    route_open_frame_with_admission_facts(
12401                        30,
12402                        "target",
12403                        unique_project_root("admission-facts"),
12404                        Some(subc_control::ConsumerIdentity {
12405                            module_id: "fed".to_string(),
12406                            launch_nonce: "fed-nonce".to_string(),
12407                        }),
12408                        None,
12409                    ),
12410                )
12411                .await
12412                .unwrap()
12413        });
12414
12415        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
12416        // the very mutation it exists to catch -- a guard that admits nobody -- no
12417        // bind is ever sent, so it HUNG rather than failing. That reproduces the
12418        // exact defect being fixed: a total revocation detected only as a stalled
12419        // suite, which reads as flakiness and invites a retry. An acceptance test
12420        // that waits for an effect must bound the wait, or a red becomes a hang.
12421        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12422            .await
12423            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12424            .expect("module control channel closed before route.bind");
12425        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12426        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12427            panic!("expected route.bind")
12428        };
12429        assert_eq!(
12430            principal,
12431            Some(Principal::Reserved {
12432                module_id: "fed".to_string()
12433            }),
12434            "a correctly attested consumer must be stamped Reserved for its own id"
12435        );
12436
12437        handler
12438            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12439            .await
12440            .unwrap();
12441        assert!(route_task.await.unwrap().is_empty());
12442        assert!(
12443            matches!(
12444                serde_json::from_slice::<ClientControlResponse>(
12445                    &client_rx.recv().await.unwrap().body
12446                )
12447                .unwrap(),
12448                ClientControlResponse::RouteOpen { .. }
12449            ),
12450            "the route must actually open, not merely avoid an error"
12451        );
12452    }
12453
12454    #[tokio::test(start_paused = true)]
12455    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12456        let registry = Arc::new(Registry::default());
12457        let forwarding = Arc::new(ForwardingTable::default());
12458        let supervisor = SupervisorHandle::new();
12459        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12460        let handler =
12461            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12462                .with_supervisor(supervisor);
12463
12464        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12465        hello_via_sink(
12466            &handler,
12467            &target_ctx,
12468            &mut target_rx,
12469            hello_frame("target", PROTOCOL_VERSION, 1),
12470        )
12471        .await;
12472
12473        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12474        let direct_handler = handler.clone();
12475        let direct_open = tokio::spawn(async move {
12476            direct_handler
12477                .handle_control_frame(
12478                    &direct_ctx,
12479                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
12480                )
12481                .await
12482                .unwrap()
12483        });
12484        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12485            .await
12486            .expect("no direct route.bind within 5s")
12487            .expect("target control channel closed before direct route.bind");
12488        handler
12489            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12490            .await
12491            .unwrap();
12492        assert!(direct_open.await.unwrap().is_empty());
12493        let _ = direct_rx.recv().await.unwrap();
12494
12495        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12496        let reserved_handler = handler.clone();
12497        let reserved_open = tokio::spawn(async move {
12498            reserved_handler
12499                .handle_control_frame(
12500                    &reserved_ctx,
12501                    route_open_frame_with_admission_facts(
12502                        3,
12503                        "target",
12504                        unique_project_root("admission-facts"),
12505                        Some(ConsumerIdentity {
12506                            module_id: "fed".to_string(),
12507                            launch_nonce: "fed-nonce".to_string(),
12508                        }),
12509                        None,
12510                    ),
12511                )
12512                .await
12513                .unwrap()
12514        });
12515        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12516            .await
12517            .expect("no reserved route.bind within 5s")
12518            .expect("target control channel closed before reserved route.bind");
12519        handler
12520            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12521            .await
12522            .unwrap();
12523        assert!(reserved_open.await.unwrap().is_empty());
12524        let _ = reserved_rx.recv().await.unwrap();
12525
12526        forwarding
12527            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12528            .unwrap();
12529        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12530        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12531            module_id: Some("target".to_string()),
12532        })
12533        .unwrap();
12534        let census_frame =
12535            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12536        let response = handler
12537            .handle_control_frame(&census_ctx, census_frame)
12538            .await
12539            .unwrap()
12540            .pop()
12541            .unwrap();
12542        let actual: Value = serde_json::from_slice(&response.body).unwrap();
12543        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12544        assert!(matches!(
12545            decoded,
12546            ClientControlResponse::SupervisorRoutes { .. }
12547        ));
12548        let routes = actual["modules"][0]["routes"].as_array().unwrap();
12549        assert_eq!(routes.len(), 2);
12550        assert!(routes.iter().all(|route| route["draining"] == true));
12551        // The census carries WHY: the reason the drain was begun with, in the
12552        // route.closing vocabulary, on every draining route this drain marked.
12553        assert!(
12554            routes.iter().all(|route| route["drain_reason"] == "reload"),
12555            "draining routes must name the drain's reason: {routes:?}"
12556        );
12557        assert!(routes.iter().any(|route| {
12558            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12559        }));
12560        assert!(routes.iter().any(|route| {
12561            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12562        }));
12563
12564        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12565            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12566        if std::env::var_os("UPDATE_GOLDEN").is_some() {
12567            std::fs::write(
12568                &golden_path,
12569                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12570            )
12571            .unwrap();
12572        }
12573        let expected: Value =
12574            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12575        assert_eq!(actual, expected);
12576    }
12577
12578    async fn query_live_roots(
12579        handler: &ControlHandler,
12580        module_ctx: &RouteCtx,
12581    ) -> ModuleControlResponseToModule {
12582        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12583        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12584        let response = handler
12585            .handle_control_frame(module_ctx, frame)
12586            .await
12587            .unwrap()
12588            .pop()
12589            .unwrap();
12590        serde_json::from_slice(&response.body).unwrap()
12591    }
12592
12593    #[tokio::test(start_paused = true)]
12594    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12595        let registry = Arc::new(Registry::default());
12596        let forwarding = Arc::new(ForwardingTable::default());
12597        let handler = ControlHandler::with_forwarding(registry, forwarding);
12598        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12599        hello_via_sink(
12600            &handler,
12601            &target_ctx,
12602            &mut target_rx,
12603            hello_frame("target", PROTOCOL_VERSION, 1),
12604        )
12605        .await;
12606        let root = unique_project_root("live-roots-known");
12607        let path = ProjectRootId::from_path_allowing_missing(root.path())
12608            .unwrap()
12609            .as_path()
12610            .to_path_buf();
12611        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12612        let open_handler = handler.clone();
12613        let opened = tokio::spawn(async move {
12614            open_handler
12615                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12616                .await
12617                .unwrap()
12618        });
12619        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12620            .await
12621            .unwrap()
12622            .unwrap();
12623        handler
12624            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12625            .await
12626            .unwrap();
12627        assert!(opened.await.unwrap().is_empty());
12628        let _ = client_rx.recv().await.unwrap();
12629
12630        let root = unique_project_root("live-roots-pending");
12631        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12632            .unwrap()
12633            .as_path()
12634            .to_path_buf();
12635        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12636        let open_handler = handler.clone();
12637        let pending = tokio::spawn(async move {
12638            open_handler
12639                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12640                .await
12641                .unwrap()
12642        });
12643        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12644            .await
12645            .unwrap()
12646            .unwrap();
12647        let actual = query_live_roots(&handler, &target_ctx).await;
12648        let ModuleControlResponseToModule::LiveRoots {
12649            roots,
12650            unknown_root_bindings,
12651            total_bindings,
12652        } = actual
12653        else {
12654            panic!("expected live roots")
12655        };
12656        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12657        assert_eq!(unknown_root_bindings, 0);
12658        assert_eq!(
12659            roots.len(),
12660            2,
12661            "root-known arm must retain each canonical root"
12662        );
12663        assert_eq!(
12664            total_bindings,
12665            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12666        );
12667        let counts = roots
12668            .iter()
12669            .map(|root| (root.project_root.clone(), root.bound, root.pending))
12670            .collect::<Vec<_>>();
12671        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12672        expected.sort_by(|a, b| a.0.cmp(&b.0));
12673        assert_eq!(
12674            counts, expected,
12675            "roots must sort by path and count pending separately"
12676        );
12677        handler
12678            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12679            .await
12680            .unwrap();
12681        assert!(pending.await.unwrap().is_empty());
12682    }
12683
12684    #[tokio::test(start_paused = true)]
12685    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12686        let forwarding = Arc::new(ForwardingTable::default());
12687        let handler =
12688            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12689        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12690        hello_via_sink(
12691            &handler,
12692            &target_ctx,
12693            &mut target_rx,
12694            hello_frame("target", PROTOCOL_VERSION, 1),
12695        )
12696        .await;
12697        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12698        let pending = forwarding
12699            .begin_route_bind_relay_for_test(
12700                client_ctx.connection_id,
12701                client_ctx.egress.clone(),
12702                2,
12703                "target",
12704            )
12705            .unwrap();
12706        forwarding
12707            .complete_pending_relay(
12708                target_ctx.connection_id,
12709                pending.corr,
12710                RouteBindRelayOutcome::Accepted,
12711            )
12712            .unwrap();
12713        let actual = query_live_roots(&handler, &target_ctx).await;
12714        let ModuleControlResponseToModule::LiveRoots {
12715            roots,
12716            unknown_root_bindings,
12717            total_bindings,
12718        } = actual
12719        else {
12720            panic!("expected live roots")
12721        };
12722        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12723        assert_eq!(
12724            unknown_root_bindings, 1,
12725            "unknown-root arm must not read as no bindings"
12726        );
12727        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12728        assert_eq!(
12729            total_bindings,
12730            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12731        );
12732    }
12733
12734    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12735    /// so the ack has to be on its outbound queue before the module is
12736    /// routable. The connection loop writes a handler's replies only after the
12737    /// handler returns; this test stops in exactly that gap, runs a real
12738    /// route.open from another connection, and only then writes whatever the
12739    /// HELLO handler returned, the way the loop would. If the ack were still a
12740    /// reply, the route.bind request would reach the module first.
12741    #[tokio::test(start_paused = true)]
12742    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12743        let forwarding = Arc::new(ForwardingTable::default());
12744        let handler =
12745            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12746        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12747        let replies = handler
12748            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12749            .await
12750            .unwrap();
12751        let queued_by_hello = module_rx.len();
12752
12753        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12754        let open_handler = handler.clone();
12755        let open = tokio::spawn(async move {
12756            open_handler
12757                .handle_control_frame(
12758                    &client_ctx,
12759                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12760                )
12761                .await
12762                .unwrap()
12763        });
12764        // Let the route.open run until its route.bind is on the module's queue.
12765        let mut spins = 0;
12766        while module_rx.len() == queued_by_hello {
12767            spins += 1;
12768            assert!(spins < 10_000, "route.open never queued a route.bind");
12769            tokio::task::yield_now().await;
12770        }
12771
12772        // Now the connection loop's half: write the HELLO handler's replies.
12773        for reply in replies {
12774            module_ctx.egress.send(reply).await.unwrap();
12775        }
12776
12777        let first = module_rx.recv().await.unwrap().frame;
12778        assert_eq!(
12779            first.header.ty,
12780            FrameType::HelloAck,
12781            "the first frame a registering module reads must be its HELLO_ACK"
12782        );
12783        assert_eq!(first.header.corr, 7);
12784        let second = module_rx.recv().await.unwrap().frame;
12785        assert_eq!(second.header.ty, FrameType::Request);
12786        assert!(
12787            matches!(
12788                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12789                ModuleControlRequest::RouteBind { .. }
12790            ),
12791            "the route.bind follows the ack"
12792        );
12793        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12794
12795        handler
12796            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12797            .await
12798            .unwrap();
12799        assert!(open.await.unwrap().is_empty());
12800        let _ = client_rx.recv().await.unwrap();
12801    }
12802
12803    #[tokio::test(start_paused = true)]
12804    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12805        let handler = ControlHandler::with_forwarding(
12806            Arc::new(Registry::default()),
12807            Arc::new(ForwardingTable::default()),
12808        );
12809        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12810        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12811        hello_via_sink(
12812            &handler,
12813            &first_ctx,
12814            &mut first_rx,
12815            hello_frame("first", PROTOCOL_VERSION, 1),
12816        )
12817        .await;
12818        hello_via_sink(
12819            &handler,
12820            &second_ctx,
12821            &mut second_rx,
12822            hello_frame("second", PROTOCOL_VERSION, 2),
12823        )
12824        .await;
12825        let root = unique_project_root("second-only");
12826        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12827        let cloned = handler.clone();
12828        let open = tokio::spawn(async move {
12829            cloned
12830                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12831                .await
12832                .unwrap()
12833        });
12834        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12835            .await
12836            .unwrap()
12837            .unwrap();
12838        let first = query_live_roots(&handler, &first_ctx).await;
12839        let second = query_live_roots(&handler, &second_ctx).await;
12840        assert!(
12841            matches!(
12842                first,
12843                ModuleControlResponseToModule::LiveRoots {
12844                    total_bindings: 0,
12845                    ..
12846                }
12847            ),
12848            "cross-module scope must not expose another module's roots"
12849        );
12850        assert!(
12851            matches!(
12852                second,
12853                ModuleControlResponseToModule::LiveRoots {
12854                    total_bindings: 1,
12855                    ..
12856                }
12857            ),
12858            "second module must see its pending route"
12859        );
12860        handler
12861            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12862            .await
12863            .unwrap();
12864        assert!(open.await.unwrap().is_empty());
12865    }
12866
12867    #[tokio::test(start_paused = true)]
12868    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12869        let handler = ControlHandler::with_forwarding(
12870            Arc::new(Registry::default()),
12871            Arc::new(ForwardingTable::default()),
12872        );
12873        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12874        hello_via_sink(
12875            &handler,
12876            &target_ctx,
12877            &mut target_rx,
12878            hello_frame("target", PROTOCOL_VERSION, 1),
12879        )
12880        .await;
12881        let actual = query_live_roots(&handler, &target_ctx).await;
12882        let ModuleControlResponseToModule::LiveRoots {
12883            roots,
12884            unknown_root_bindings,
12885            total_bindings,
12886        } = actual
12887        else {
12888            panic!("expected live roots")
12889        };
12890        assert!(roots.is_empty());
12891        assert_eq!(unknown_root_bindings, 0);
12892        assert_eq!(total_bindings, 0);
12893        assert_eq!(
12894            total_bindings,
12895            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12896        );
12897    }
12898
12899    /// Read the vendored fed corpus rather than hand-building a package.
12900    ///
12901    /// A hand-built object encodes what the test author believed the carrier
12902    /// emits. These vectors are what it actually emits, and one of them exists
12903    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
12904    /// ignores additive unknown fields at the traversal emit terminus."
12905    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12906        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12907            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12908        let text = std::fs::read_to_string(&path)
12909            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12910        let vectors: Vec<(String, Value)> = text
12911            .lines()
12912            .filter(|line| !line.trim().is_empty())
12913            .map(|line| {
12914                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12915                let id = entry["corpus_id"]
12916                    .as_str()
12917                    .expect("every vector carries a corpus_id")
12918                    .to_string();
12919                (id, entry["package"].clone())
12920            })
12921            .collect();
12922        // Pin the count: a corpus that silently shrinks would take its coverage
12923        // with it, and a suite reading N-1 vectors reports the same clean pass
12924        // as one reading N.
12925        assert_eq!(
12926            vectors.len(),
12927            3,
12928            "vendored fed corpus changed size; re-sync from subc-federation"
12929        );
12930
12931        // Pin what makes the corpus DISCRIMINATING, not just present.
12932        //
12933        // The relay test below takes its expected value from the corpus, so the
12934        // corpus supplies the test's power to detect a lossy relay rather than
12935        // its correctness. A relay that dropped unrecognised fields would still
12936        // be caught -- but only by a package carrying fields it does not know.
12937        // Shrink every package to the handful of keys any implementation would
12938        // recognise and the test keeps passing over an input that can no longer
12939        // fail, which is the same clean green as a corpus that shrank away.
12940        //
12941        // So assert the precondition rather than duplicating the packages here:
12942        // at least one vector must carry a field beyond the small common set.
12943        // That is one claim to maintain instead of nine, and it fails loudly if
12944        // a re-sync ever flattens the corpus.
12945        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12946        let richest = vectors
12947            .iter()
12948            .filter_map(|(_, package)| package.as_object())
12949            .map(|object| {
12950                object
12951                    .keys()
12952                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12953                    .count()
12954            })
12955            .max()
12956            .unwrap_or(0);
12957        assert!(
12958            richest >= 2,
12959            "vendored corpus no longer carries a package with unmodelled fields, \
12960             so the relay test can no longer distinguish a verbatim relay from a lossy one"
12961        );
12962
12963        vectors
12964    }
12965
12966    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
12967    ///
12968    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
12969    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
12970    /// three-key object, so a relay that quietly dropped fields it did not
12971    /// recognise would satisfy it. These vectors carry nine keys including ones
12972    /// this crate has no type for, so a typed relay fails here and only here.
12973    #[tokio::test]
12974    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12975        for (corpus_id, package) in fed_admission_facts_vectors() {
12976            let registry = Arc::new(Registry::default());
12977            let forwarding = Arc::new(ForwardingTable::default());
12978            let supervisor = SupervisorHandle::new();
12979            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12980            let handler =
12981                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12982                    .with_supervisor(supervisor)
12983                    .with_admission_facts_config(
12984                        Some("fed".to_string()),
12985                        Some(vec!["target".to_string()]),
12986                    );
12987
12988            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12989            hello_via_sink(
12990                &handler,
12991                &target_ctx,
12992                &mut target_rx,
12993                hello_frame("target", PROTOCOL_VERSION, 1),
12994            )
12995            .await;
12996
12997            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12998            let route_handler = handler.clone();
12999            let expected = package.clone();
13000            let route_task = tokio::spawn(async move {
13001                route_handler
13002                    .handle_control_frame(
13003                        &client_ctx,
13004                        route_open_frame_with_admission_facts(
13005                            20,
13006                            "target",
13007                            unique_project_root("admission-facts"),
13008                            Some(subc_control::ConsumerIdentity {
13009                                module_id: "fed".to_string(),
13010                                launch_nonce: "fed-nonce".to_string(),
13011                            }),
13012                            Some(package),
13013                        ),
13014                    )
13015                    .await
13016                    .unwrap()
13017            });
13018
13019            let bind_frame = target_rx.recv().await.unwrap();
13020            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13021            let ModuleControlRequest::RouteBind {
13022                admission_facts, ..
13023            } = bind
13024            else {
13025                panic!("{corpus_id}: expected route.bind")
13026            };
13027            assert_eq!(
13028                admission_facts,
13029                Some(expected),
13030                "{corpus_id}: relay must not add, drop or reshape any field"
13031            );
13032
13033            handler
13034                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13035                .await
13036                .unwrap();
13037            route_task.await.unwrap();
13038        }
13039    }
13040
13041    #[tokio::test]
13042    async fn route_bind_never_relays_the_openers_build_class() {
13043        let registry = Arc::new(Registry::default());
13044        let forwarding = Arc::new(ForwardingTable::default());
13045        let handler =
13046            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13047        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13048        hello_via_sink(
13049            &handler,
13050            &target_ctx,
13051            &mut target_rx,
13052            hello_frame("target", PROTOCOL_VERSION, 1),
13053        )
13054        .await;
13055
13056        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13057        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13058        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13059            busy_refusals: None,
13060            target: RouteTarget::ToolProvider {
13061                module_id: "target".to_string(),
13062            },
13063            identity,
13064            consumer_identity: None,
13065            consumer_capabilities: None,
13066            role_versions: None,
13067            admission_facts: None,
13068            scope: None,
13069        })
13070        .unwrap();
13071        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13072
13073        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13074        let route_handler = handler.clone();
13075        let route_task = tokio::spawn(async move {
13076            route_handler
13077                .handle_control_frame(&client_ctx, open)
13078                .await
13079                .unwrap()
13080        });
13081        let bind_frame = target_rx.recv().await.unwrap();
13082        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13083        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13084            panic!("expected route.bind")
13085        };
13086        assert_eq!(
13087            identity.build_class, None,
13088            "route.bind carried the build class"
13089        );
13090        assert!(
13091            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13092            "the route.bind bytes still name build_class"
13093        );
13094        handler
13095            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13096            .await
13097            .unwrap();
13098        route_task.await.unwrap();
13099    }
13100
13101    #[tokio::test]
13102    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13103        let registry = Arc::new(Registry::default());
13104        let forwarding = Arc::new(ForwardingTable::default());
13105        let supervisor = SupervisorHandle::new();
13106        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13107        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13108        let handler =
13109            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13110                .with_supervisor(supervisor)
13111                .with_admission_facts_config(
13112                    Some("fed".to_string()),
13113                    Some(vec!["target".to_string()]),
13114                );
13115
13116        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13117        hello_via_sink(
13118            &handler,
13119            &target_ctx,
13120            &mut target_rx,
13121            hello_frame("target", PROTOCOL_VERSION, 1),
13122        )
13123        .await;
13124        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13125        hello_via_sink(
13126            &handler,
13127            &other_ctx,
13128            &mut other_rx,
13129            hello_frame("other", PROTOCOL_VERSION, 2),
13130        )
13131        .await;
13132
13133        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13134        let expected_facts = facts.clone();
13135        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13136        let route_handler = handler.clone();
13137        let route_task = tokio::spawn(async move {
13138            route_handler
13139                .handle_control_frame(
13140                    &client_ctx,
13141                    route_open_frame_with_admission_facts(
13142                        10,
13143                        "target",
13144                        unique_project_root("admission-facts"),
13145                        Some(subc_control::ConsumerIdentity {
13146                            module_id: "fed".to_string(),
13147                            launch_nonce: "fed-nonce".to_string(),
13148                        }),
13149                        Some(facts.clone()),
13150                    ),
13151                )
13152                .await
13153                .unwrap()
13154        });
13155        let bind_frame = target_rx.recv().await.unwrap();
13156        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13157        let ModuleControlRequest::RouteBind {
13158            admission_facts, ..
13159        } = bind
13160        else {
13161            panic!("expected route.bind")
13162        };
13163        assert_eq!(admission_facts, Some(expected_facts));
13164        handler
13165            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13166            .await
13167            .unwrap();
13168        assert!(route_task.await.unwrap().is_empty());
13169        assert!(matches!(
13170            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13171                .unwrap(),
13172            ClientControlResponse::RouteOpen { .. }
13173        ));
13174
13175        let direct = handler
13176            .handle_control_frame(
13177                &route_ctx(ConnectionId::new(73)).0,
13178                route_open_frame_with_admission_facts(
13179                    11,
13180                    "target",
13181                    unique_project_root("admission-facts"),
13182                    None,
13183                    Some(json!({"x": 1})),
13184                ),
13185            )
13186            .await
13187            .unwrap();
13188        assert_eq!(
13189            parse_error(&direct[0])["code"],
13190            "admission_facts_not_permitted"
13191        );
13192
13193        let different_reserved = handler
13194            .handle_control_frame(
13195                &route_ctx(ConnectionId::new(77)).0,
13196                route_open_frame_with_admission_facts(
13197                    15,
13198                    "target",
13199                    unique_project_root("admission-facts"),
13200                    Some(subc_control::ConsumerIdentity {
13201                        module_id: "other".to_string(),
13202                        launch_nonce: "other-nonce".to_string(),
13203                    }),
13204                    Some(json!({"x": 1})),
13205                ),
13206            )
13207            .await
13208            .unwrap();
13209        assert_eq!(
13210            parse_error(&different_reserved[0])["code"],
13211            "admission_facts_not_permitted"
13212        );
13213
13214        let other_target = handler
13215            .handle_control_frame(
13216                &route_ctx(ConnectionId::new(74)).0,
13217                route_open_frame_with_admission_facts(
13218                    12,
13219                    "other",
13220                    unique_project_root("admission-facts"),
13221                    Some(subc_control::ConsumerIdentity {
13222                        module_id: "fed".to_string(),
13223                        launch_nonce: "fed-nonce".to_string(),
13224                    }),
13225                    Some(json!({"x": 1})),
13226                ),
13227            )
13228            .await
13229            .unwrap();
13230        assert_eq!(
13231            parse_error(&other_target[0])["code"],
13232            "admission_facts_target_not_allowed"
13233        );
13234
13235        let nonexistent = handler
13236            .handle_control_frame(
13237                &route_ctx(ConnectionId::new(75)).0,
13238                route_open_frame_with_admission_facts(
13239                    13,
13240                    "missing",
13241                    unique_project_root("admission-facts"),
13242                    None,
13243                    Some(json!({"x": 1})),
13244                ),
13245            )
13246            .await
13247            .unwrap();
13248        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13249
13250        let described = handler
13251            .handle_control_frame(
13252                &route_ctx(ConnectionId::new(76)).0,
13253                Frame::build(
13254                    FrameType::Request,
13255                    control_flags(),
13256                    0,
13257                    0,
13258                    14,
13259                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13260                )
13261                .unwrap(),
13262            )
13263            .await
13264            .unwrap();
13265        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13266            serde_json::from_slice(&described[0].body).unwrap()
13267        else {
13268            panic!("expected server.describe response")
13269        };
13270        assert!(capabilities
13271            .iter()
13272            .any(|cap| cap == "admission_facts_relay_v1"));
13273    }
13274
13275    #[tokio::test]
13276    async fn admission_facts_without_configured_carrier_are_rejected() {
13277        let registry = Arc::new(Registry::default());
13278        let forwarding = Arc::new(ForwardingTable::default());
13279        let handler = ControlHandler::with_forwarding(registry, forwarding);
13280        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13281        hello_via_sink(
13282            &handler,
13283            &target_ctx,
13284            &mut target_rx,
13285            hello_frame("target", PROTOCOL_VERSION, 1),
13286        )
13287        .await;
13288
13289        let responses = handler
13290            .handle_control_frame(
13291                &route_ctx(ConnectionId::new(79)).0,
13292                route_open_frame_with_admission_facts(
13293                    16,
13294                    "target",
13295                    unique_project_root("admission-facts"),
13296                    None,
13297                    Some(json!({"x": 1})),
13298                ),
13299            )
13300            .await
13301            .unwrap();
13302        assert_eq!(
13303            parse_error(&responses[0])["code"],
13304            "admission_facts_not_permitted"
13305        );
13306    }
13307
13308    #[tokio::test]
13309    async fn route_open_relays_consumer_capabilities_verbatim() {
13310        let registry = Arc::new(Registry::default());
13311        let forwarding = Arc::new(ForwardingTable::default());
13312        let handler =
13313            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13314        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13315        hello_via_sink(
13316            &handler,
13317            &module_ctx,
13318            &mut module_rx,
13319            hello_frame("aft", PROTOCOL_VERSION, 7),
13320        )
13321        .await;
13322
13323        let expected = vec!["elicitation".to_string(), "roots".to_string()];
13324        let expected_for_request = expected.clone();
13325        let project_root = unique_project_root("consumer-capabilities-present");
13326        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13327        let route_handler = handler.clone();
13328        let route_task = tokio::spawn(async move {
13329            route_handler
13330                .handle_control_frame(
13331                    &client_ctx,
13332                    route_open_frame_with_consumer_capabilities(
13333                        401,
13334                        "aft",
13335                        project_root,
13336                        Some(expected_for_request),
13337                    ),
13338                )
13339                .await
13340                .unwrap()
13341        });
13342        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13343            .await
13344            .unwrap()
13345            .unwrap();
13346        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13347        let ModuleControlRequest::RouteBind {
13348            consumer_capabilities,
13349            ..
13350        } = bind
13351        else {
13352            panic!("expected route.bind request, got {bind:?}");
13353        };
13354        assert_eq!(consumer_capabilities, Some(expected.clone()));
13355
13356        handler
13357            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13358            .await
13359            .unwrap();
13360        let route_response = route_task.await.unwrap();
13361        assert!(route_response.is_empty());
13362        let published = client_rx.recv().await.unwrap();
13363        assert!(matches!(
13364            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13365            ClientControlResponse::RouteOpen { .. }
13366        ));
13367    }
13368
13369    #[tokio::test]
13370    async fn route_open_without_consumer_capabilities_relays_none() {
13371        let registry = Arc::new(Registry::default());
13372        let forwarding = Arc::new(ForwardingTable::default());
13373        let handler =
13374            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13375        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13376        hello_via_sink(
13377            &handler,
13378            &module_ctx,
13379            &mut module_rx,
13380            hello_frame("aft", PROTOCOL_VERSION, 7),
13381        )
13382        .await;
13383
13384        let project_root = unique_project_root("consumer-capabilities-absent");
13385        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13386        let route_handler = handler.clone();
13387        let route_task = tokio::spawn(async move {
13388            route_handler
13389                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13390                .await
13391                .unwrap()
13392        });
13393        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13394            .await
13395            .unwrap()
13396            .unwrap();
13397        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13398        let ModuleControlRequest::RouteBind {
13399            consumer_capabilities,
13400            ..
13401        } = bind
13402        else {
13403            panic!("expected route.bind request, got {bind:?}");
13404        };
13405        assert_eq!(consumer_capabilities, None);
13406
13407        handler
13408            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13409            .await
13410            .unwrap();
13411        let route_response = route_task.await.unwrap();
13412        assert!(route_response.is_empty());
13413        let published = client_rx.recv().await.unwrap();
13414        assert!(matches!(
13415            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13416            ClientControlResponse::RouteOpen { .. }
13417        ));
13418    }
13419
13420    /// Opens a route to a freshly registered `aft` with `sent` as the
13421    /// route.open's role_versions, acks the bind, and returns the role_versions
13422    /// the module's bind carried.
13423    async fn bind_role_versions_for(
13424        sent: Option<BTreeMap<String, String>>,
13425        connection: u64,
13426    ) -> Option<BTreeMap<String, String>> {
13427        let registry = Arc::new(Registry::default());
13428        let forwarding = Arc::new(ForwardingTable::default());
13429        let handler =
13430            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13431        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13432        hello_via_sink(
13433            &handler,
13434            &module_ctx,
13435            &mut module_rx,
13436            hello_frame("aft", PROTOCOL_VERSION, 7),
13437        )
13438        .await;
13439        let project_root = unique_project_root("role-versions");
13440        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13441        let route_handler = handler.clone();
13442        let route_task = tokio::spawn(async move {
13443            route_handler
13444                .handle_control_frame(
13445                    &client_ctx,
13446                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
13447                )
13448                .await
13449                .unwrap()
13450        });
13451        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13452            .await
13453            .expect("a well-formed route.open reaches the module as a bind")
13454            .unwrap();
13455        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13456        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13457            panic!("expected route.bind request, got {bind:?}");
13458        };
13459        handler
13460            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13461            .await
13462            .unwrap();
13463        assert!(route_task.await.unwrap().is_empty());
13464        let published = client_rx.recv().await.unwrap();
13465        assert!(matches!(
13466            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13467            ClientControlResponse::RouteOpen { .. }
13468        ));
13469        role_versions
13470    }
13471
13472    #[tokio::test]
13473    async fn route_open_relays_role_versions_verbatim() {
13474        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13475        assert_eq!(
13476            bind_role_versions_for(Some(sent.clone()), 141).await,
13477            Some(sent)
13478        );
13479    }
13480
13481    /// An empty map declares nothing, so the provider sees no field rather
13482    /// than an empty object it would have to treat as a second "none".
13483    #[tokio::test]
13484    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13485        assert_eq!(bind_role_versions_for(None, 143).await, None);
13486        assert_eq!(
13487            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13488            None
13489        );
13490    }
13491
13492    #[tokio::test]
13493    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13494        let registry = Arc::new(Registry::default());
13495        let forwarding = Arc::new(ForwardingTable::default());
13496        let handler =
13497            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13498        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13499        hello_via_sink(
13500            &handler,
13501            &module_ctx,
13502            &mut module_rx,
13503            hello_frame("aft", PROTOCOL_VERSION, 7),
13504        )
13505        .await;
13506
13507        let nine: BTreeMap<String, String> = (0..9)
13508            .map(|index| (format!("role-{index}"), "v1".to_string()))
13509            .collect();
13510        for (label, malformed) in [
13511            (
13512                "invalid role name",
13513                role_versions(&[("Tool_Provider", "v1")]),
13514            ),
13515            ("invalid version", role_versions(&[("tool-provider", "v0")])),
13516            ("nine entries", nine),
13517        ] {
13518            // A refused open answers at once; one that reached the module
13519            // would wait for its bind ack and trip this timeout.
13520            let responses = tokio::time::timeout(
13521                Duration::from_secs(1),
13522                handler.handle_control_frame(
13523                    &route_ctx(ConnectionId::new(148)).0,
13524                    route_open_frame_with_role_versions(
13525                        404,
13526                        "aft",
13527                        unique_project_root("role-versions-malformed"),
13528                        Some(malformed),
13529                    ),
13530                ),
13531            )
13532            .await
13533            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13534            .unwrap();
13535            assert_eq!(responses.len(), 1, "{label}");
13536            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13537            let error = parse_error(&responses[0]);
13538            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13539            assert_eq!(
13540                error["detail"]["field"], "role_versions",
13541                "{label}: {error}"
13542            );
13543            assert!(
13544                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13545                "{label}: a malformed declaration is terminal"
13546            );
13547            assert!(
13548                module_rx.try_recv().is_err(),
13549                "{label}: the module must never see a bind"
13550            );
13551        }
13552    }
13553
13554    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
13555    /// consumer can tell this daemon forwards the field from one that would
13556    /// drop it.
13557    #[tokio::test]
13558    async fn route_role_versions_capability_is_advertised() {
13559        let handler = ControlHandler::new(Arc::new(Registry::default()));
13560        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13561        let ack = hello_via_sink(
13562            &handler,
13563            &ctx,
13564            &mut rx,
13565            hello_frame("m", PROTOCOL_VERSION, 1),
13566        )
13567        .await;
13568        let ack = parse_ack(&ack);
13569        assert!(
13570            ack.subc_capabilities
13571                .iter()
13572                .any(|c| c == "route-role-versions/v1"),
13573            "{:?}",
13574            ack.subc_capabilities
13575        );
13576
13577        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13578        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13579        let reply = handler
13580            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13581            .await
13582            .unwrap()
13583            .pop()
13584            .unwrap();
13585        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13586            serde_json::from_slice(&reply.body).unwrap()
13587        else {
13588            panic!("not a server.describe reply");
13589        };
13590        assert!(
13591            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13592            "{capabilities:?}"
13593        );
13594    }
13595
13596    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
13597    /// catalog entry without `events` (the module declares none) from a daemon
13598    /// that never reports declared events.
13599    #[tokio::test]
13600    async fn catalog_events_capability_is_advertised() {
13601        let handler = ControlHandler::new(Arc::new(Registry::default()));
13602        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13603        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13604        let reply = handler
13605            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13606            .await
13607            .unwrap()
13608            .pop()
13609            .unwrap();
13610        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13611            serde_json::from_slice(&reply.body).unwrap()
13612        else {
13613            panic!("not a server.describe reply");
13614        };
13615        assert!(
13616            capabilities
13617                .iter()
13618                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13619            "{capabilities:?}"
13620        );
13621    }
13622
13623    #[tokio::test]
13624    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13625        let handler = ControlHandler::new(Arc::new(Registry::default()));
13626        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13627        let ack = hello_via_sink(
13628            &handler,
13629            &module,
13630            &mut module_rx,
13631            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13632        )
13633        .await;
13634        let ack = parse_ack(&ack);
13635        assert!(
13636            ack.subc_capabilities
13637                .iter()
13638                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13639            "{:?}",
13640            ack.subc_capabilities
13641        );
13642        assert!(ack
13643            .subc_capabilities
13644            .iter()
13645            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13646
13647        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13648        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13649        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13650        let reply = handler
13651            .handle_control_frame(&client, frame)
13652            .await
13653            .unwrap()
13654            .pop()
13655            .unwrap();
13656        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13657            serde_json::from_slice(&reply.body).unwrap()
13658        else {
13659            panic!("not a server.describe reply");
13660        };
13661        assert!(
13662            capabilities
13663                .iter()
13664                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13665            "{capabilities:?}"
13666        );
13667        assert!(capabilities
13668            .iter()
13669            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13670    }
13671
13672    #[tokio::test]
13673    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13674        let handler = ControlHandler::new(Arc::new(Registry::default()));
13675        let (module, mut rx) = route_ctx(ConnectionId::new(156));
13676        let ack = hello_via_sink(
13677            &handler,
13678            &module,
13679            &mut rx,
13680            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13681        )
13682        .await;
13683        assert!(parse_ack(&ack)
13684            .subc_capabilities
13685            .iter()
13686            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13687        assert!(parse_ack(&ack)
13688            .subc_capabilities
13689            .iter()
13690            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13691        let (client, _rx) = route_ctx(ConnectionId::new(157));
13692        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13693        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13694        let reply = handler
13695            .handle_control_frame(&client, frame)
13696            .await
13697            .unwrap()
13698            .pop()
13699            .unwrap();
13700        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13701            serde_json::from_slice(&reply.body).unwrap()
13702        else {
13703            panic!("server.describe reply")
13704        };
13705        assert!(capabilities
13706            .iter()
13707            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13708        assert!(capabilities
13709            .iter()
13710            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13711    }
13712
13713    #[tokio::test]
13714    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13715        let handler = ControlHandler::new(Arc::new(Registry::default()));
13716        let (module, mut rx) = route_ctx(ConnectionId::new(158));
13717        let ack = hello_via_sink(
13718            &handler,
13719            &module,
13720            &mut rx,
13721            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13722        )
13723        .await;
13724        assert!(parse_ack(&ack)
13725            .subc_capabilities
13726            .iter()
13727            .any(|cap| cap == "scope-workdir/v1"));
13728        let (client, _rx) = route_ctx(ConnectionId::new(159));
13729        let frame = Frame::build(
13730            FrameType::Request,
13731            control_flags(),
13732            0,
13733            0,
13734            5,
13735            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13736        )
13737        .unwrap();
13738        let reply = handler
13739            .handle_control_frame(&client, frame)
13740            .await
13741            .unwrap()
13742            .remove(0);
13743        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13744            serde_json::from_slice(&reply.body).unwrap()
13745        else {
13746            panic!("server.describe reply")
13747        };
13748        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13749    }
13750
13751    #[tokio::test]
13752    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13753        let handler = ControlHandler::new(Arc::new(Registry::default()));
13754        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13755        let ack = hello_via_sink(
13756            &handler,
13757            &module,
13758            &mut rx,
13759            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13760        )
13761        .await;
13762        assert!(parse_ack(&ack)
13763            .subc_capabilities
13764            .iter()
13765            .any(|cap| cap == "worker-runs/v1"));
13766        let (client, _rx) = route_ctx(ConnectionId::new(161));
13767        let frame = Frame::build(
13768            FrameType::Request,
13769            control_flags(),
13770            0,
13771            0,
13772            5,
13773            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13774        )
13775        .unwrap();
13776        let reply = handler
13777            .handle_control_frame(&client, frame)
13778            .await
13779            .unwrap()
13780            .remove(0);
13781        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13782            serde_json::from_slice(&reply.body).unwrap()
13783        else {
13784            panic!("server.describe reply")
13785        };
13786        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13787    }
13788
13789    #[tokio::test]
13790    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13791        let handler = ControlHandler::new(Arc::new(Registry::default()));
13792        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13793        let ack = hello_via_sink(
13794            &handler,
13795            &module,
13796            &mut rx,
13797            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13798        )
13799        .await;
13800        assert!(parse_ack(&ack)
13801            .subc_capabilities
13802            .iter()
13803            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13804        let (client, _rx) = route_ctx(ConnectionId::new(155));
13805        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13806        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13807        let reply = handler
13808            .handle_control_frame(&client, frame)
13809            .await
13810            .unwrap()
13811            .pop()
13812            .unwrap();
13813        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13814            serde_json::from_slice(&reply.body).unwrap()
13815        else {
13816            panic!("not a server.describe reply")
13817        };
13818        assert!(capabilities
13819            .iter()
13820            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13821    }
13822
13823    #[tokio::test]
13824    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13825        let registry = Arc::new(Registry::default());
13826        let forwarding = Arc::new(ForwardingTable::default());
13827        let handler =
13828            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13829                .with_health_probe_timeout(Duration::from_secs(5));
13830        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13831        hello_via_sink(
13832            &handler,
13833            &module_ctx,
13834            &mut module_rx,
13835            non_routable_hello_frame_with_control_ops(
13836                "mcp",
13837                300,
13838                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13839            ),
13840        )
13841        .await;
13842        assert!(registry
13843            .get_module("mcp")
13844            .unwrap()
13845            .unwrap()
13846            .manifest
13847            .provides
13848            .is_empty());
13849
13850        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13851        let route_response = handler
13852            .handle_control_frame(
13853                &route_client_ctx,
13854                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13855            )
13856            .await
13857            .unwrap();
13858        assert_eq!(route_response[0].header.ty, FrameType::Error);
13859        assert_eq!(
13860            parse_error(&route_response[0])["code"],
13861            "target_unavailable"
13862        );
13863        assert!(parse_error(&route_response[0])["message"]
13864            .as_str()
13865            .unwrap()
13866            .contains("does not provide the requested target"));
13867        assert!(module_rx.try_recv().is_err());
13868
13869        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13870        let health_handler = handler.clone();
13871        let health_task = tokio::spawn(async move {
13872            health_handler
13873                .handle_control_frame(
13874                    &health_client_ctx,
13875                    supervisor_health_probe_frame(302, "mcp"),
13876                )
13877                .await
13878                .unwrap()
13879        });
13880        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13881            .await
13882            .unwrap()
13883            .unwrap();
13884        assert_eq!(
13885            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13886            ModuleControlRequest::HealthCheck {}
13887        );
13888        handler
13889            .handle_control_frame(
13890                &module_ctx,
13891                health_response(health_frame.header.corr, HealthStatus::Ok),
13892            )
13893            .await
13894            .unwrap();
13895        let health_response = health_task.await.unwrap();
13896        assert_eq!(health_response[0].header.ty, FrameType::Response);
13897        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13898            ClientControlResponse::SupervisorHealthProbe {
13899                module_id, status, ..
13900            } => {
13901                assert_eq!(module_id, "mcp");
13902                assert_eq!(status, HealthStatus::Ok);
13903            }
13904            other => panic!("unexpected health response: {other:?}"),
13905        }
13906
13907        // Exercise the forwarding cleanup path directly while leaving the registry
13908        // advertisement in place. If cleanup leaves a stale control sink behind,
13909        // the next probe will enqueue onto it and wait for the long probe timeout
13910        // instead of returning an immediate no-connection error.
13911        forwarding
13912            .cleanup_connection(module_ctx.connection_id)
13913            .unwrap();
13914        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13915        let cleanup_response = tokio::time::timeout(
13916            Duration::from_millis(200),
13917            handler.handle_control_frame(
13918                &cleanup_probe_ctx,
13919                supervisor_health_probe_frame(303, "mcp"),
13920            ),
13921        )
13922        .await
13923        .expect("probe should fail immediately when the control lane is gone")
13924        .unwrap();
13925        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13926        assert_eq!(
13927            parse_error(&cleanup_response[0])["code"],
13928            "target_unavailable"
13929        );
13930        assert!(parse_error(&cleanup_response[0])["message"]
13931            .as_str()
13932            .unwrap()
13933            .contains("no module connection"));
13934
13935        handler
13936            .cleanup_connection(module_ctx.connection_id)
13937            .unwrap();
13938    }
13939
13940    #[tokio::test]
13941    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13942        let registry = Arc::new(Registry::default());
13943        let supervisor_handle = SupervisorHandle::new();
13944        let supervisor =
13945            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13946                .with_handle(supervisor_handle.clone())
13947                .with_connection_file_path(
13948                    std::env::temp_dir()
13949                        .join(format!("subc-route-open-warming-{}", std::process::id())),
13950                );
13951        let module = supervisor
13952            .supervise_configured(
13953                ModuleSpec {
13954                    carry_containment: false,
13955                    module_id: "warming".to_string(),
13956                    program: fake_aft_stub_path(),
13957                    args: Vec::new(),
13958                    env: Vec::new(),
13959                    reserved: false,
13960                    reserved_prefixes: Vec::new(),
13961                    protocol: ModuleProtocol::Subc,
13962                    overlap: Default::default(),
13963                },
13964                true,
13965            )
13966            .unwrap();
13967        assert_eq!(module.state().unwrap(), ModuleState::Running);
13968
13969        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13970        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13971        let response = handler
13972            .handle_control_frame(
13973                &ctx,
13974                route_open_frame(304, "warming", unique_project_root("warming")),
13975            )
13976            .await
13977            .unwrap();
13978        module.stop().await.unwrap();
13979
13980        assert_eq!(response[0].header.ty, FrameType::Error);
13981        let error = parse_error(&response[0]);
13982        assert_eq!(error["code"], "module_warming");
13983        assert!(error["message"]
13984            .as_str()
13985            .unwrap()
13986            .contains("state=running, enabled=true, live=false"));
13987    }
13988
13989    #[test]
13990    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13991        let handler = ControlHandler::new(Arc::new(Registry::default()));
13992        let capture = EventCapture::default();
13993        let _subscriber =
13994            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13995        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13996        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13997        let pending = (0..limit).collect::<Vec<_>>();
13998        let response = handler
13999            .route_open_capacity_refusal(
14000                &ctx,
14001                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14002                "busy",
14003                pending.len(),
14004                limit,
14005            )
14006            .unwrap();
14007        assert_eq!(parse_error(&response)["code"], "target_unavailable");
14008        let event = capture
14009            .events()
14010            .into_iter()
14011            .find(|event| {
14012                event.target == "control"
14013                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14014            })
14015            .expect("connection admission refusal event");
14016        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14017        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14018        // Refused before `handle_route_open` parses the request, so the caller
14019        // fields are read from the frame itself.
14020        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14021        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14022    }
14023
14024    #[test]
14025    fn route_open_target_cap_logs_admission_reason_and_capacity() {
14026        let handler = ControlHandler::new(Arc::new(Registry::default()));
14027        let capture = EventCapture::default();
14028        let _subscriber =
14029            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14030        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14031        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14032        let guards = (0..limit)
14033            .map(|_| {
14034                handler
14035                    .route_bind_concurrency
14036                    .try_admit("busy", limit)
14037                    .unwrap()
14038            })
14039            .collect::<Vec<_>>();
14040        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14041            Err(in_flight) => in_flight,
14042            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14043        };
14044        let response = handler
14045            .route_open_target_capacity_refusal(
14046                &ctx,
14047                &RouteOpenCaller::unknown(),
14048                &route_open_frame(397, "busy", unique_project_root("target-cap")),
14049                "busy",
14050                in_flight,
14051            )
14052            .unwrap();
14053        assert_eq!(parse_error(&response)["code"], "target_unavailable");
14054        let event = capture
14055            .events()
14056            .into_iter()
14057            .find(|event| {
14058                event.target == "control"
14059                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14060            })
14061            .expect("target admission refusal event");
14062        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14063        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14064        drop(guards);
14065    }
14066
14067    #[cfg(unix)]
14068    #[tokio::test(start_paused = true)]
14069    async fn supervisor_list_shows_a_failed_modules_next_retry() {
14070        let registry = Arc::new(Registry::default());
14071        let handle = SupervisorHandle::new();
14072        let supervisor =
14073            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14074                .with_handle(handle.clone());
14075        let module = supervisor
14076            .spawn(crate::ModuleSpec {
14077                carry_containment: false,
14078                module_id: "retry-list".into(),
14079                program: PathBuf::from("/bin/sh"),
14080                args: vec!["-c".into(), "exit 17".into()],
14081                env: vec![],
14082                reserved: false,
14083                reserved_prefixes: vec![],
14084                protocol: ModuleProtocol::None,
14085                overlap: Default::default(),
14086            })
14087            .unwrap();
14088        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14089        while module.state().unwrap() != crate::ModuleState::Failed {
14090            assert!(
14091                std::time::Instant::now() < deadline,
14092                "waiting for budget failure"
14093            );
14094            tokio::task::yield_now().await;
14095        }
14096        let handler = ControlHandler::new(registry).with_supervisor(handle);
14097        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14098        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14099        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14100        assert_eq!(body["modules"][0]["state"], "failed");
14101        let retry = body["modules"][0]["next_retry_at_ms"]
14102            .as_u64()
14103            .expect("scheduled retry on the wire");
14104        let now = std::time::SystemTime::now()
14105            .duration_since(std::time::UNIX_EPOCH)
14106            .unwrap()
14107            .as_millis() as u64;
14108        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14109        module.set_enabled(false).await.unwrap();
14110        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14111        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14112        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14113        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14114    }
14115
14116    #[tokio::test]
14117    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14118        let _guard = crate::subtree_resources::fault_read_guard().await;
14119        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14120        let supervisor =
14121            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14122        let module = supervisor
14123            .spawn(crate::ModuleSpec {
14124                carry_containment: false,
14125                module_id: "pid-read-budget".into(),
14126                program: std::env::current_exe().unwrap(),
14127                args: vec![
14128                    "--exact".into(),
14129                    "control::tests::pid_resource_read_fixture".into(),
14130                ],
14131                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14132                    .into_iter()
14133                    .map(|key| (key.into(), dir.display().to_string()))
14134                    .chain(std::iter::once((
14135                        "SUBC_SUBTREE_PID_FIXTURE".into(),
14136                        "1".into(),
14137                    )))
14138                    .collect(),
14139                reserved: false,
14140                reserved_prefixes: Vec::new(),
14141                protocol: ModuleProtocol::None,
14142                overlap: Default::default(),
14143            })
14144            .unwrap();
14145        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14146        while !matches!(
14147            module.child_resource_usage(),
14148            subc_control::ChildResourceUsage::Measured(_)
14149        ) {
14150            assert!(
14151                tokio::time::Instant::now() < ready_deadline,
14152                "fixture process was not measurable"
14153            );
14154            tokio::time::sleep(Duration::from_millis(10)).await;
14155        }
14156        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14157        let reading = crate::subtree_resources::read_process(module.clone()).await;
14158        module.stop().await.unwrap();
14159        assert!(
14160            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14161            "{reading:?}"
14162        );
14163    }
14164
14165    #[test]
14166    fn pid_resource_read_fixture() {
14167        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14168            loop {
14169                std::thread::park_timeout(Duration::from_secs(600));
14170            }
14171        }
14172    }
14173
14174    #[tokio::test]
14175    async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14176        let registry = Arc::new(Registry::default());
14177        let handle = SupervisorHandle::new();
14178        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14179            .with_handle(handle.clone());
14180        supervisor
14181            .supervise_configured(
14182                crate::ModuleSpec {
14183                    module_id: "disabled-resource-reading".into(),
14184                    program: std::env::current_exe().unwrap(),
14185                    args: Vec::new(),
14186                    env: Vec::new(),
14187                    reserved: false,
14188                    reserved_prefixes: Vec::new(),
14189                    protocol: ModuleProtocol::None,
14190                    overlap: Default::default(),
14191                    carry_containment: false,
14192                },
14193                false,
14194            )
14195            .unwrap();
14196        let handler = ControlHandler::new(registry).with_supervisor(handle);
14197        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14198        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14199        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14200        assert!(body["modules"][0].get("tree").is_none());
14201        assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14202    }
14203
14204    #[tokio::test]
14205    async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14206        let _guard = crate::subtree_resources::fault_read_guard().await;
14207        let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14208        let registry = Arc::new(Registry::default());
14209        let handle = SupervisorHandle::new();
14210        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14211            .with_handle(handle.clone());
14212        let module = supervisor
14213            .spawn(crate::ModuleSpec {
14214                carry_containment: false,
14215                module_id: "list-process-priority".into(),
14216                program: std::env::current_exe().unwrap(),
14217                args: vec![
14218                    "--exact".into(),
14219                    "control::tests::pid_resource_read_fixture".into(),
14220                ],
14221                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14222                    .into_iter()
14223                    .map(|key| (key.into(), dir.display().to_string()))
14224                    .chain(std::iter::once((
14225                        "SUBC_SUBTREE_PID_FIXTURE".into(),
14226                        "1".into(),
14227                    )))
14228                    .collect(),
14229                reserved: false,
14230                reserved_prefixes: Vec::new(),
14231                protocol: ModuleProtocol::None,
14232                overlap: Default::default(),
14233            })
14234            .unwrap();
14235        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14236        loop {
14237            let image = module.running_image_agreement().await;
14238            if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14239                break;
14240            }
14241            assert!(
14242                tokio::time::Instant::now() < ready_deadline,
14243                "the fixture process never became observable: {image:?}"
14244            );
14245            tokio::time::sleep(Duration::from_millis(10)).await;
14246        }
14247        let (target, release, _) = crate::subtree_resources::held();
14248        module.set_subtree_target_for_test(target);
14249        let handler = ControlHandler::new(registry).with_supervisor(handle);
14250        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14251        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14252        drop(release);
14253        module.stop().await.unwrap();
14254        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14255        assert_eq!(
14256            body["modules"][0]["resources"]["status"], "measured",
14257            "the optional tree timeout must not discard the process reading: {body}"
14258        );
14259        assert!(body["modules"][0].get("tree").is_none());
14260        assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14261        assert_eq!(
14262            body["modules"][0]["pending_reload"]["image"]["status"], "match",
14263            "the tree timeout must not hide a completed image observation"
14264        );
14265    }
14266
14267    #[cfg(any(target_os = "linux", target_os = "macos"))]
14268    #[tokio::test]
14269    async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14270        let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14271            .with_daemon_provenance(
14272                std::process::id(),
14273                0,
14274                Some(std::env::current_exe().unwrap()),
14275                None,
14276                None,
14277            );
14278        let (probe, release, began) = ExecutableIdentityProbe::held();
14279        handler.daemon_provenance.probe = probe;
14280        let observe = |handler: ControlHandler, corr| {
14281            tokio::spawn(async move {
14282                let (ctx, _) = route_ctx(ConnectionId::new(corr));
14283                let frame =
14284                    Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14285                        .unwrap();
14286                handler
14287                    .handle_supervisor_provenance(&ctx, frame, None)
14288                    .await
14289                    .unwrap()
14290            })
14291        };
14292        let explicit = observe(handler.clone(), 114);
14293        began.await.unwrap();
14294        let concurrent = observe(handler.clone(), 115);
14295        // Keep explicit provenance waiting on its first image observation beyond
14296        // the list deadline; release it only after the concurrent list replies.
14297        tokio::time::sleep(Duration::from_millis(300)).await;
14298        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14299        let list = tokio::time::timeout(
14300            Duration::from_secs(1),
14301            handler.handle_supervisor_list(frame),
14302        )
14303        .await;
14304        let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14305        drop(release);
14306        let replies = explicit.await.unwrap();
14307        let concurrent_replies = concurrent.await.unwrap();
14308        assert!(
14309            explicit_was_pending,
14310            "explicit provenance must await the full observation"
14311        );
14312        let list = list
14313            .expect("a concurrent explicit probe must not stall list")
14314            .unwrap();
14315        let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14316        assert_eq!(list_body["op"], "supervisor.list");
14317        for replies in [replies, concurrent_replies] {
14318            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14319            assert_eq!(
14320                body["daemon"]["daemon_observed"]["running_image"]["status"],
14321                "match"
14322            );
14323        }
14324    }
14325
14326    #[tokio::test]
14327    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14328        let _guard = crate::subtree_resources::fault_read_guard().await;
14329        let registry = Arc::new(Registry::default());
14330        let handle = SupervisorHandle::new();
14331        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14332            .with_handle(handle.clone());
14333        let module = supervisor
14334            .supervise_configured(
14335                crate::ModuleSpec {
14336                    carry_containment: false,
14337                    module_id: "memory-read".into(),
14338                    program: PathBuf::from("disabled-module"),
14339                    args: Vec::new(),
14340                    env: Vec::new(),
14341                    reserved: false,
14342                    reserved_prefixes: Vec::new(),
14343                    protocol: ModuleProtocol::Subc,
14344                    overlap: Default::default(),
14345                },
14346                true,
14347            )
14348            .unwrap();
14349        let (target, release, began) = crate::subtree_resources::held();
14350        // Optional list work may be skipped when its deadline is already spent.
14351        // Establish the stalled fixture before asking the list to observe it.
14352        let stalled_target = target.clone();
14353        let stalled = tokio::task::spawn_blocking(move || {
14354            stalled_target.tree(&subc_os::ProcessTable::default())
14355        });
14356        began.await.unwrap();
14357        module.set_subtree_target_for_test(target);
14358        let handler = ControlHandler::new(registry).with_supervisor(handle);
14359        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14360        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14361        let replies = handler
14362            .handle_client_control_request(
14363                &ctx,
14364                frame,
14365                ClientControlRequest::SupervisorList {},
14366                &mut None,
14367            )
14368            .await
14369            .unwrap();
14370        assert!(
14371            replies.is_empty(),
14372            "list reply must be detached from the connection reader"
14373        );
14374        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14375        let replies = handler
14376            .handle_client_control_request(
14377                &ctx,
14378                frame,
14379                ClientControlRequest::SupervisorSpawnSnapshot {},
14380                &mut None,
14381            )
14382            .await
14383            .unwrap();
14384        assert_eq!(
14385            replies.len(),
14386            1,
14387            "another request must complete while the subtree read stalls"
14388        );
14389        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14390            .await
14391            .unwrap()
14392            .unwrap();
14393        release.send(()).unwrap();
14394        drop(release);
14395        stalled.await.unwrap();
14396        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14397        assert!(
14398            body["modules"][0].get("tree").is_none(),
14399            "an unknown tree reading must be omitted after the bounded read expires"
14400        );
14401        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14402        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14403        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14404        assert_eq!(
14405            body["modules"][0]["tree"]["source"], "process_tree",
14406            "the next request must read afresh"
14407        );
14408        assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14409    }
14410
14411    #[tokio::test]
14412    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14413        let registry = Arc::new(Registry::default());
14414        let handle = SupervisorHandle::new();
14415        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14416            .with_handle(handle.clone());
14417        supervisor
14418            .supervise_configured(
14419                crate::ModuleSpec {
14420                    carry_containment: false,
14421                    module_id: "drift".into(),
14422                    program: PathBuf::from("disabled-module"),
14423                    args: Vec::new(),
14424                    env: Vec::new(),
14425                    reserved: false,
14426                    reserved_prefixes: Vec::new(),
14427                    protocol: ModuleProtocol::Subc,
14428                    overlap: Default::default(),
14429                },
14430                false,
14431            )
14432            .unwrap();
14433        let handler = ControlHandler::new(registry).with_supervisor(handle);
14434        for configured in [None, Some(false), Some(true)] {
14435            handler.capability_evaluator.configure(
14436                configured.map(|enabled| ("drift".to_string(), enabled)),
14437                BTreeMap::new(),
14438            );
14439            let frame =
14440                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14441            let replies = handler
14442                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14443                .await
14444                .unwrap();
14445            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14446            assert!(body.get("configured_enabled").is_none());
14447            assert_eq!(body["op"], "supervisor.ack");
14448            let frame =
14449                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14450            let replies = handler.handle_supervisor_list(frame).await.unwrap();
14451            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14452            assert_eq!(
14453                body["modules"][0].get("configured_enabled"),
14454                configured.map(Value::Bool).as_ref()
14455            );
14456            assert!(body["modules"][0].get("tree").is_none());
14457            assert!(body["modules"][0].get("subtree").is_none());
14458        }
14459    }
14460
14461    #[tokio::test(flavor = "current_thread")]
14462    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14463        let handler = ControlHandler::new(Arc::new(Registry::default()));
14464        let capture = EventCapture::default();
14465        let _subscriber =
14466            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14467        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14468        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14469
14470        let _ = handler
14471            .handle_client_control_request(
14472                &ctx,
14473                frame,
14474                ClientControlRequest::SupervisorSetEnabled {
14475                    module_id: "broca".to_string(),
14476                    enabled: false,
14477                },
14478                &mut None,
14479            )
14480            .await
14481            .unwrap();
14482
14483        let events = capture
14484            .events()
14485            .into_iter()
14486            .filter(|event| {
14487                event.target == "control"
14488                    && event.fields.get("message").map(String::as_str)
14489                        == Some("supervisor request received")
14490            })
14491            .collect::<Vec<_>>();
14492        assert_eq!(events.len(), 1, "one supervisor request log line");
14493        let event = &events[0];
14494        assert_eq!(event.level, tracing::Level::INFO);
14495        assert_eq!(
14496            event.fields.get("op"),
14497            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14498        );
14499        assert_eq!(
14500            event.fields.get("module_id"),
14501            Some(&"\"broca\"".to_string())
14502        );
14503        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14504        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14505        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14506    }
14507
14508    #[tokio::test(flavor = "current_thread")]
14509    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14510        let registry = Arc::new(Registry::default());
14511        let module_connection = ConnectionId::new(303);
14512        registry
14513            .register_with_control_ops(
14514                manifest("aft", PROTOCOL_VERSION),
14515                PROTOCOL_VERSION,
14516                module_connection,
14517                module_baseline_control_ops(),
14518            )
14519            .unwrap();
14520        let handler = ControlHandler::new(registry);
14521        let capture = EventCapture::default();
14522        let _subscriber =
14523            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14524        let (ctx, _rx) = route_ctx(module_connection);
14525        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14526
14527        let _ = handler
14528            .handle_client_control_request(
14529                &ctx,
14530                frame,
14531                ClientControlRequest::SupervisorSetEnabled {
14532                    module_id: "broca".to_string(),
14533                    enabled: false,
14534                },
14535                &mut None,
14536            )
14537            .await
14538            .unwrap();
14539
14540        let callers = capture
14541            .events()
14542            .into_iter()
14543            .filter(|event| {
14544                event.fields.get("message").map(String::as_str)
14545                    == Some("supervisor request received")
14546            })
14547            .map(|event| event.fields.get("caller").cloned())
14548            .collect::<Vec<_>>();
14549        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14550    }
14551
14552    #[tokio::test(flavor = "current_thread")]
14553    async fn supervisor_list_does_not_log_request_received() {
14554        let handler = ControlHandler::new(Arc::new(Registry::default()));
14555        let capture = EventCapture::default();
14556        let _subscriber =
14557            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14558        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14559        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14560
14561        handler
14562            .handle_client_control_request(
14563                &ctx,
14564                frame,
14565                ClientControlRequest::SupervisorList {},
14566                &mut None,
14567            )
14568            .await
14569            .unwrap();
14570
14571        assert!(capture.events().into_iter().all(|event| {
14572            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14573        }));
14574    }
14575
14576    #[tokio::test(flavor = "current_thread")]
14577    async fn supervisor_rescan_preview_does_not_log_request_received() {
14578        let handler = ControlHandler::new(Arc::new(Registry::default()));
14579        let capture = EventCapture::default();
14580        let _subscriber =
14581            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14582        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14583        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14584
14585        handler
14586            .handle_client_control_request(
14587                &ctx,
14588                frame,
14589                ClientControlRequest::SupervisorRescan { preview: true },
14590                &mut None,
14591            )
14592            .await
14593            .unwrap();
14594
14595        assert!(capture.events().into_iter().all(|event| {
14596            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14597        }));
14598    }
14599
14600    struct ApproveOperator;
14601    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14602        fn prompt(
14603            &self,
14604            _: &str,
14605            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14606        ) -> crate::operator_confirm::ProviderResult {
14607            crate::operator_confirm::ProviderResult::Approved
14608        }
14609    }
14610
14611    #[tokio::test]
14612    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14613        assert_eq!(
14614            ControlHandler::default().privileged_operator_prompt,
14615            PrivilegedOperatorPrompt::Log
14616        );
14617        for (initial, changed, running) in [
14618            ("log", "enforce", PrivilegedOperatorPrompt::Log),
14619            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14620        ] {
14621            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14622            let path = dir.join("subc.jsonc");
14623            std::fs::write(
14624                &path,
14625                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14626            )
14627            .unwrap();
14628            let config = crate::daemon_config::load(&path).unwrap().unwrap();
14629            let registry = Arc::new(Registry::default());
14630            let handle = SupervisorHandle::new();
14631            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14632                .with_handle(handle.clone());
14633            let handler = ControlHandler::new(registry)
14634                .with_supervisor(handle)
14635                .with_privileged_operator_prompt(config.privileged_operator_prompt)
14636                .with_supervisor_rescan(supervisor, &path, config.port);
14637            handler.forwarding.operator_confirms().configure(
14638                Arc::new(ApproveOperator),
14639                crate::operator_confirm::OperatorLimits::default(),
14640            );
14641            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14642
14643            for (disk_mode, needs_restart) in [
14644                (Some(initial), false),
14645                (Some(changed), true),
14646                (Some(changed), true),
14647                (None, running == PrivilegedOperatorPrompt::Enforce),
14648                (Some(initial), false),
14649            ] {
14650                let mut doc = json!({"version":1});
14651                if let Some(mode) = disk_mode {
14652                    doc["privileged_operator_prompt"] = json!(mode);
14653                }
14654                std::fs::write(&path, doc.to_string()).unwrap();
14655                for preview in [true, false] {
14656                    let request = ClientControlRequest::SupervisorRescan { preview };
14657                    let frame = Frame::build(
14658                        FrameType::Request,
14659                        control_flags(),
14660                        0,
14661                        0,
14662                        1,
14663                        serde_json::to_vec(&request).unwrap(),
14664                    )
14665                    .unwrap();
14666                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14667                    assert_eq!(frames.len(), 1);
14668                    let ClientControlResponse::SupervisorRescan { result } =
14669                        serde_json::from_slice(&frames[0].body).unwrap()
14670                    else {
14671                        panic!("expected a rescan response");
14672                    };
14673                    let expected = if needs_restart {
14674                        vec!["privileged_operator_prompt"]
14675                    } else {
14676                        Vec::new()
14677                    };
14678                    assert_eq!(result.restart_required, expected);
14679                    assert_eq!(handler.privileged_operator_prompt, running);
14680                    assert_eq!(
14681                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14682                        running
14683                    );
14684                }
14685            }
14686        }
14687    }
14688
14689    #[tokio::test]
14690    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14691        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14692        let path = dir.join("subc.jsonc");
14693        let registry = Arc::new(Registry::default());
14694        let handle = SupervisorHandle::new();
14695        let supervisor =
14696            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14697        let handler = ControlHandler::new(registry)
14698            .with_supervisor(handle)
14699            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14700            .with_supervisor_rescan(supervisor, &path, None);
14701        handler.forwarding.operator_confirms().configure(
14702            Arc::new(ApproveOperator),
14703            crate::operator_confirm::OperatorLimits::default(),
14704        );
14705        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14706        for preview in [true, false] {
14707            std::fs::write(
14708                &path,
14709                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14710            )
14711            .unwrap();
14712            let request = ClientControlRequest::SupervisorRescan { preview };
14713            let frame = Frame::build(
14714                FrameType::Request,
14715                control_flags(),
14716                0,
14717                0,
14718                1,
14719                serde_json::to_vec(&request).unwrap(),
14720            )
14721            .unwrap();
14722            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14723            assert_eq!(frames.len(), 1);
14724            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14725            assert_eq!(
14726                handler.privileged_operator_prompt,
14727                PrivilegedOperatorPrompt::Enforce
14728            );
14729        }
14730    }
14731
14732    /// One wire code has several senders, so the refusal line names the check
14733    /// that refused. This drives the shared refusal path for ordinary refusals
14734    /// with an unregistered
14735    /// target and requires the branch label on the event.
14736    #[tokio::test]
14737    async fn route_open_refusal_names_the_check_that_refused() {
14738        let handler = ControlHandler::new(Arc::new(Registry::default()));
14739        let capture = EventCapture::default();
14740        let _subscriber =
14741            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14742        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14743        let response = handler
14744            .handle_control_frame(
14745                &ctx,
14746                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14747            )
14748            .await
14749            .unwrap();
14750
14751        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14752        let event = capture
14753            .events()
14754            .into_iter()
14755            .find(|event| {
14756                event.target == "control"
14757                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14758            })
14759            .expect("route.open refusal event");
14760        assert_eq!(
14761            event.fields.get("reason"),
14762            Some(&"\"not_registered\"".to_string())
14763        );
14764    }
14765
14766    #[tokio::test]
14767    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14768        let registry = Arc::new(Registry::default());
14769        let supervisor_handle = SupervisorHandle::new();
14770        let supervisor =
14771            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14772                .with_handle(supervisor_handle.clone())
14773                .with_connection_file_path(std::env::temp_dir().join(format!(
14774                    "subc-route-open-refusal-info-{}",
14775                    std::process::id()
14776                )));
14777        let module = supervisor
14778            .supervise_configured(
14779                ModuleSpec {
14780                    carry_containment: false,
14781                    module_id: "warming".to_string(),
14782                    program: fake_aft_stub_path(),
14783                    args: Vec::new(),
14784                    env: Vec::new(),
14785                    reserved: false,
14786                    reserved_prefixes: Vec::new(),
14787                    protocol: ModuleProtocol::Subc,
14788                    overlap: Default::default(),
14789                },
14790                true,
14791            )
14792            .unwrap();
14793        assert_eq!(module.state().unwrap(), ModuleState::Running);
14794
14795        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14796        assert!(handler
14797            .counters()
14798            .snapshot()
14799            .get("route_open_refused_by_code")
14800            .is_none());
14801        let capture = EventCapture::default();
14802        let _subscriber =
14803            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14804        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14805        let response = handler
14806            .handle_control_frame(
14807                &ctx,
14808                route_open_frame(394, "warming", unique_project_root("refusal-info")),
14809            )
14810            .await
14811            .unwrap();
14812        module.stop().await.unwrap();
14813
14814        assert_eq!(parse_error(&response[0])["code"], "module_warming");
14815        let event = capture
14816            .events()
14817            .into_iter()
14818            .find(|event| {
14819                event.target == "control"
14820                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14821            })
14822            .expect("route.open refusal event");
14823        assert_eq!(
14824            event.fields.get("module_id"),
14825            Some(&"\"warming\"".to_string())
14826        );
14827        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14828        assert_eq!(
14829            event.fields.get("reason"),
14830            Some(&"\"supervised_not_registered\"".to_string())
14831        );
14832        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14833        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14834        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14835        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14836        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14837        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14838        assert_eq!(
14839            handler.counters().snapshot()["route_open_refused_by_code"],
14840            json!({ "module_warming": 1 })
14841        );
14842    }
14843
14844    const OUTAGE_START: &str = "route.open refusing module: not serving";
14845    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14846
14847    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14848        capture
14849            .events()
14850            .into_iter()
14851            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14852            .collect()
14853    }
14854
14855    fn supervise_stub(
14856        registry: &Arc<Registry>,
14857        module_id: &str,
14858        enabled: bool,
14859    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14860        let supervisor_handle = SupervisorHandle::new();
14861        let supervisor =
14862            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14863                .with_handle(supervisor_handle.clone())
14864                .with_connection_file_path(std::env::temp_dir().join(format!(
14865                    "subc-route-outage-{module_id}-{}",
14866                    std::process::id()
14867                )));
14868        let module = supervisor
14869            .supervise_configured(
14870                ModuleSpec {
14871                    carry_containment: false,
14872                    module_id: module_id.to_string(),
14873                    program: fake_aft_stub_path(),
14874                    args: Vec::new(),
14875                    env: Vec::new(),
14876                    reserved: false,
14877                    reserved_prefixes: Vec::new(),
14878                    protocol: ModuleProtocol::Subc,
14879                    overlap: Default::default(),
14880                },
14881                enabled,
14882            )
14883            .unwrap();
14884        (supervisor_handle, module)
14885    }
14886
14887    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14888        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14889            module_id: module_id.to_string(),
14890            drain_timeout_ms: Some(50),
14891        })
14892        .unwrap();
14893        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14894    }
14895
14896    /// Two handlers built over one forwarding table must share one outage
14897    /// tracker; separate trackers would each log their own opening line for
14898    /// the same outage.
14899    #[test]
14900    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14901        let registry = Arc::new(Registry::default());
14902        let forwarding = Arc::new(ForwardingTable::default());
14903        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14904        let second = ControlHandler::with_forwarding(registry, forwarding);
14905        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14906    }
14907
14908    /// A client can name any module id it likes. Refusing an unknown one,
14909    /// however often, must not create outage state or outage lines, or the
14910    /// tracker would be a memory sink any client could fill.
14911    #[tokio::test(flavor = "current_thread")]
14912    async fn route_open_unknown_module_refusals_add_no_outage_state() {
14913        let handler = ControlHandler::new(Arc::new(Registry::default()));
14914        let capture = EventCapture::default();
14915        let _subscriber =
14916            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14917        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14918        for corr in 0..8 {
14919            let response = handler
14920                .handle_control_frame(
14921                    &ctx,
14922                    route_open_frame(
14923                        380 + corr,
14924                        &format!("nobody-{corr}"),
14925                        unique_project_root("outage-unknown"),
14926                    ),
14927                )
14928                .await
14929                .unwrap();
14930            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14931        }
14932
14933        assert_eq!(handler.route_outages.tracked_module_count(), 0);
14934        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14935        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14936    }
14937
14938    /// Drives the refusal path end to end: a supervised module that served
14939    /// before and stopped being registered with no instruction to stop is a
14940    /// WARN, and the same module refused after an operator `supervisor.restart`
14941    /// is an INFO.
14942    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14943    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14944        let registry = Arc::new(Registry::default());
14945        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
14946        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14947        let capture = EventCapture::default();
14948        let _subscriber =
14949            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14950        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14951        // The stub never registers, so pretend it served once: otherwise every
14952        // refusal would fall in its startup window.
14953        handler.route_outages.record_accepted("outage-restart");
14954
14955        let response = handler
14956            .handle_control_frame(
14957                &ctx,
14958                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14959            )
14960            .await
14961            .unwrap();
14962        assert_eq!(response[0].header.ty, FrameType::Error);
14963        let starts = outage_lines(&capture, OUTAGE_START);
14964        assert_eq!(starts.len(), 1, "{starts:?}");
14965        assert_eq!(starts[0].level, tracing::Level::WARN);
14966        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14967        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14968        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14969        handler.route_outages.record_accepted("outage-restart");
14970        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14971
14972        let restart = handler
14973            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14974            .await
14975            .unwrap();
14976        assert_eq!(
14977            restart[0].header.ty,
14978            FrameType::Response,
14979            "{:?}",
14980            parse_error(&restart[0])
14981        );
14982        handler
14983            .handle_control_frame(
14984                &ctx,
14985                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14986            )
14987            .await
14988            .unwrap();
14989        module.stop().await.unwrap();
14990
14991        let starts = outage_lines(&capture, OUTAGE_START);
14992        assert_eq!(starts.len(), 2, "{starts:?}");
14993        assert_eq!(starts[1].level, tracing::Level::INFO);
14994        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14995    }
14996
14997    /// A restart refused before it touched the module (here: the module is
14998    /// disabled) must clear its operator mark, so the next real outage is
14999    /// still reported as a warning.
15000    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15001    async fn failed_operator_restart_leaves_no_operator_mark() {
15002        let registry = Arc::new(Registry::default());
15003        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
15004        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15005        let capture = EventCapture::default();
15006        let _subscriber =
15007            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15008        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15009        handler.route_outages.record_accepted("outage-disabled");
15010
15011        let restart = handler
15012            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15013            .await
15014            .unwrap();
15015        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15016        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15017
15018        handler
15019            .handle_control_frame(
15020                &ctx,
15021                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15022            )
15023            .await
15024            .unwrap();
15025        let starts = outage_lines(&capture, OUTAGE_START);
15026        assert_eq!(starts.len(), 1, "{starts:?}");
15027        assert_eq!(starts[0].level, tracing::Level::WARN);
15028    }
15029
15030    #[tokio::test(flavor = "current_thread")]
15031    async fn route_open_unknown_module_escapes_target_module_id() {
15032        let handler = ControlHandler::new(Arc::new(Registry::default()));
15033        let capture = EventCapture::default();
15034        let _subscriber =
15035            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15036        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15037        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15038        let response = handler
15039            .handle_control_frame(
15040                &ctx,
15041                route_open_frame(
15042                    395,
15043                    hostile_module_id,
15044                    unique_project_root("hostile-target-module-id"),
15045                ),
15046            )
15047            .await
15048            .unwrap();
15049
15050        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15051        let event = capture
15052            .events()
15053            .into_iter()
15054            .find(|event| {
15055                event.target == "control"
15056                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15057            })
15058            .expect("route.open unknown-module refusal event");
15059        let logged = event.fields.get("module_id").expect("module_id field");
15060        assert!(!logged.bytes().any(|byte| byte < 0x20));
15061        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15062    }
15063
15064    /// A route.open refusal is the daemon's own answer, so its ERROR carries
15065    /// the daemon-origin bit on the frame the client receives.
15066    #[tokio::test(flavor = "current_thread")]
15067    async fn route_open_refusal_carries_daemon_origin() {
15068        let handler = ControlHandler::new(Arc::new(Registry::default()));
15069        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15070        let response = handler
15071            .handle_control_frame(
15072                &ctx,
15073                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15074            )
15075            .await
15076            .unwrap();
15077
15078        assert_eq!(response.len(), 1);
15079        assert_eq!(response[0].header.ty, FrameType::Error);
15080        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15081        assert!(
15082            response[0].header.flags.is_daemon_origin(),
15083            "route.open refusal must carry daemon origin: {:?}",
15084            response[0].header.flags
15085        );
15086    }
15087
15088    /// The drain `module_reloading` refusal is daemon-synthesized for an
15089    /// endpoint that is reloading, so it carries the daemon-origin bit on the
15090    /// frame the client receives.
15091    #[tokio::test(flavor = "current_thread")]
15092    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15093        let registry = Arc::new(Registry::default());
15094        let forwarding = Arc::new(ForwardingTable::default());
15095        let handler =
15096            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15097        let module_connection = ConnectionId::new(95);
15098        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15099        hello_via_sink(
15100            &handler,
15101            &module_ctx,
15102            &mut module_rx,
15103            hello_frame("aft", PROTOCOL_VERSION, 395),
15104        )
15105        .await;
15106        forwarding
15107            .begin_module_drain("aft", RouteCloseReason::Restart)
15108            .unwrap()
15109            .expect("the module endpoint drains");
15110
15111        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15112        let response = handler
15113            .handle_control_frame(
15114                &client_ctx,
15115                route_open_frame(396, "aft", unique_project_root("drain-origin")),
15116            )
15117            .await
15118            .unwrap();
15119
15120        assert_eq!(response.len(), 1);
15121        assert_eq!(response[0].header.ty, FrameType::Error);
15122        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15123        assert!(
15124            response[0].header.flags.is_daemon_origin(),
15125            "drain module_reloading must carry daemon origin: {:?}",
15126            response[0].header.flags
15127        );
15128    }
15129
15130    /// A module that refuses a bind with a daemon code still cannot mint a
15131    /// daemon refusal: the relayed `module_reloading` body reaches the client
15132    /// with the daemon-origin bit clear.
15133    #[tokio::test(flavor = "current_thread")]
15134    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15135        let registry = Arc::new(Registry::default());
15136        let forwarding = Arc::new(ForwardingTable::default());
15137        let handler =
15138            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15139        let module_connection = ConnectionId::new(95);
15140        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15141        hello_via_sink(
15142            &handler,
15143            &module_ctx,
15144            &mut module_rx,
15145            hello_frame("aft", PROTOCOL_VERSION, 395),
15146        )
15147        .await;
15148
15149        let client_connection = ConnectionId::new(96);
15150        let (client_ctx, _client_rx) = route_ctx(client_connection);
15151        let (route_task, bind) = relay_route_open(
15152            &handler,
15153            client_connection,
15154            &client_ctx.egress,
15155            &mut module_rx,
15156            396,
15157            "aft",
15158            "spoofed-reloading",
15159        )
15160        .await;
15161        // The module sets the origin bit on its own rejection too: neither
15162        // the bit nor the daemon code may survive the relay as daemon-owned.
15163        let rejection = Frame::build(
15164            FrameType::Error,
15165            crate::router::daemon_error_flags(),
15166            0,
15167            0,
15168            bind.header.corr,
15169            serde_json::to_vec(&ErrorBody::new(
15170                "module_reloading",
15171                "module refused route.bind",
15172            ))
15173            .unwrap(),
15174        )
15175        .unwrap();
15176        handler
15177            .handle_control_frame(&module_ctx, rejection)
15178            .await
15179            .unwrap();
15180
15181        let response = route_task.await.unwrap();
15182        assert_eq!(response.len(), 1);
15183        assert_eq!(response[0].header.ty, FrameType::Error);
15184        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15185        assert!(
15186            !response[0].header.flags.is_daemon_origin(),
15187            "a module's own rejection must not carry daemon origin: {:?}",
15188            response[0].header.flags
15189        );
15190    }
15191
15192    #[tokio::test(flavor = "current_thread")]
15193    async fn route_open_module_rejection_uses_daemon_counter_key() {
15194        let registry = Arc::new(Registry::default());
15195        let forwarding = Arc::new(ForwardingTable::default());
15196        let handler =
15197            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15198        let module_connection = ConnectionId::new(95);
15199        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15200        hello_via_sink(
15201            &handler,
15202            &module_ctx,
15203            &mut module_rx,
15204            hello_frame("aft", PROTOCOL_VERSION, 395),
15205        )
15206        .await;
15207
15208        let client_connection = ConnectionId::new(96);
15209        let (client_ctx, _client_rx) = route_ctx(client_connection);
15210        let capture = EventCapture::default();
15211        let _subscriber =
15212            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15213        let (route_task, bind) = relay_route_open(
15214            &handler,
15215            client_connection,
15216            &client_ctx.egress,
15217            &mut module_rx,
15218            396,
15219            "aft",
15220            "hostile-module-code",
15221        )
15222        .await;
15223        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15224        let rejection = Frame::build(
15225            FrameType::Error,
15226            control_flags(),
15227            0,
15228            0,
15229            bind.header.corr,
15230            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15231        )
15232        .unwrap();
15233        handler
15234            .handle_control_frame(&module_ctx, rejection)
15235            .await
15236            .unwrap();
15237
15238        let response = route_task.await.unwrap();
15239        assert_eq!(parse_error(&response[0])["code"], hostile_code);
15240        let counters = handler.counters().snapshot();
15241        assert_eq!(
15242            counters["route_open_refused_by_code"],
15243            json!({ "module_rejected": 1 })
15244        );
15245        assert!(counters["route_open_refused_by_code"]
15246            .get(hostile_code)
15247            .is_none());
15248
15249        let event = capture
15250            .events()
15251            .into_iter()
15252            .find(|event| {
15253                event.target == "control"
15254                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15255            })
15256            .expect("route.open module-rejection refusal event");
15257        let logged = event.fields.get("module_code").expect("module_code field");
15258        assert!(!logged.bytes().any(|byte| byte < 0x20));
15259        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15260    }
15261
15262    #[tokio::test]
15263    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15264        let registry = Arc::new(Registry::default());
15265        let supervisor_handle = SupervisorHandle::new();
15266        let missing_program = std::env::temp_dir().join(format!(
15267            "subc-route-open-missing-program-{}",
15268            std::process::id()
15269        ));
15270        let supervisor =
15271            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
15272                .with_handle(supervisor_handle.clone());
15273        let module = supervisor
15274            .supervise_configured(
15275                ModuleSpec {
15276                    carry_containment: false,
15277                    module_id: "failed".to_string(),
15278                    program: missing_program,
15279                    args: Vec::new(),
15280                    env: Vec::new(),
15281                    reserved: false,
15282                    reserved_prefixes: Vec::new(),
15283                    protocol: ModuleProtocol::Subc,
15284                    overlap: Default::default(),
15285                },
15286                true,
15287            )
15288            .unwrap();
15289        assert_eq!(module.state().unwrap(), ModuleState::Failed);
15290
15291        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15292        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15293        let response = handler
15294            .handle_control_frame(
15295                &ctx,
15296                route_open_frame(305, "failed", unique_project_root("failed")),
15297            )
15298            .await
15299            .unwrap();
15300
15301        assert_eq!(response[0].header.ty, FrameType::Error);
15302        let error = parse_error(&response[0]);
15303        assert_eq!(error["code"], "target_unavailable");
15304        assert!(error["message"]
15305            .as_str()
15306            .unwrap()
15307            .contains("state=failed, enabled=true, live=false"));
15308    }
15309
15310    #[tokio::test]
15311    async fn route_open_role_mismatch_remains_target_unavailable() {
15312        let registry = Arc::new(Registry::default());
15313        let handler = ControlHandler::new(Arc::clone(&registry));
15314        handler
15315            .handle_control(
15316                ConnectionId::new(41),
15317                non_routable_hello_frame_with_control_ops("health-only", 306, None),
15318            )
15319            .unwrap();
15320
15321        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15322        let response = handler
15323            .handle_control_frame(
15324                &ctx,
15325                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15326            )
15327            .await
15328            .unwrap();
15329
15330        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15331        assert!(parse_error(&response[0])["message"]
15332            .as_str()
15333            .unwrap()
15334            .contains("does not provide the requested target"));
15335    }
15336
15337    #[tokio::test]
15338    async fn route_open_inactive_registration_remains_target_unavailable() {
15339        let registry = Arc::new(Registry::default());
15340        let handler = ControlHandler::new(Arc::clone(&registry));
15341        handler
15342            .handle_control(
15343                ConnectionId::new(43),
15344                hello_frame("inactive", PROTOCOL_VERSION, 308),
15345            )
15346            .unwrap();
15347        assert!(registry
15348            .set_module_state_for_test("inactive", ChannelState::Closed)
15349            .unwrap());
15350
15351        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15352        let response = handler
15353            .handle_control_frame(
15354                &ctx,
15355                route_open_frame(309, "inactive", unique_project_root("inactive")),
15356            )
15357            .await
15358            .unwrap();
15359
15360        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15361        assert!(parse_error(&response[0])["message"]
15362            .as_str()
15363            .unwrap()
15364            .contains("is not active"));
15365    }
15366
15367    #[tokio::test]
15368    async fn late_health_reply_is_recorded_through_the_module_response_path() {
15369        let registry = Arc::new(Registry::default());
15370        let forwarding = Arc::new(ForwardingTable::default());
15371        let supervisor_handle = SupervisorHandle::new();
15372        let supervisor =
15373            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
15374                .with_forwarding(Arc::clone(&forwarding))
15375                .with_handle(supervisor_handle.clone());
15376        let module = supervisor
15377            .supervise_configured(
15378                crate::ModuleSpec {
15379                    carry_containment: false,
15380                    module_id: "late-health-response".to_string(),
15381                    program: PathBuf::from("disabled-module"),
15382                    args: Vec::new(),
15383                    env: Vec::new(),
15384                    reserved: false,
15385                    reserved_prefixes: Vec::new(),
15386                    protocol: ModuleProtocol::Subc,
15387                    overlap: Default::default(),
15388                },
15389                false,
15390            )
15391            .unwrap();
15392        let handler =
15393            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15394                .with_supervisor(supervisor_handle);
15395        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15396        handler
15397            .handle_control_frame(
15398                &module_ctx,
15399                hello_frame_with_control_ops(
15400                    "late-health-response",
15401                    PROTOCOL_VERSION,
15402                    7,
15403                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15404                ),
15405            )
15406            .await
15407            .unwrap();
15408        let probe_started_at = Instant::now() - Duration::from_millis(80);
15409        let pending = forwarding
15410            .begin_health_probe_rpc_for(
15411                "late-health-response",
15412                MODULE_CONTROL_OP_HEALTH_CHECK,
15413                probe_started_at,
15414                Instant::now() - Duration::from_millis(1),
15415            )
15416            .unwrap();
15417        assert!(forwarding
15418            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15419            .unwrap());
15420
15421        let responses = handler
15422            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15423            .await
15424            .unwrap();
15425
15426        assert!(responses.is_empty());
15427        let health = module.status().unwrap().health;
15428        assert_eq!(health.late_answer_count, 1);
15429        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15430    }
15431
15432    #[tokio::test]
15433    async fn health_probe_timeout_and_module_death_are_typed() {
15434        let registry = Arc::new(Registry::default());
15435        let forwarding = Arc::new(ForwardingTable::default());
15436        let handler =
15437            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15438                .with_health_probe_timeout(Duration::from_millis(50));
15439        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15440        hello_via_sink(
15441            &handler,
15442            &module_ctx,
15443            &mut module_rx,
15444            hello_frame_with_control_ops(
15445                "aft",
15446                PROTOCOL_VERSION,
15447                7,
15448                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15449            ),
15450        )
15451        .await;
15452
15453        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15454        let responses = handler
15455            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15456            .await
15457            .unwrap();
15458        assert_eq!(responses[0].header.ty, FrameType::Error);
15459        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15460        let _ = module_rx.try_recv();
15461
15462        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15463        let health_handler = handler.clone();
15464        let death_task = tokio::spawn(async move {
15465            health_handler
15466                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15467                .await
15468                .unwrap()
15469        });
15470        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15471            .await
15472            .unwrap()
15473            .unwrap();
15474        handler
15475            .cleanup_connection(module_ctx.connection_id)
15476            .unwrap();
15477        let responses = death_task.await.unwrap();
15478        assert_eq!(responses[0].header.ty, FrameType::Error);
15479        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15480    }
15481
15482    #[test]
15483    fn hello_requires_exact_protocol_version() {
15484        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15485            let registry = Arc::new(Registry::default());
15486            let handler = ControlHandler::new(Arc::clone(&registry));
15487            let responses = handler
15488                .handle_control(
15489                    ConnectionId::new(connection),
15490                    hello_frame("aft", offered, 9),
15491                )
15492                .unwrap();
15493
15494            assert_eq!(responses.len(), 1);
15495            assert_eq!(responses[0].header.ty, FrameType::Error);
15496            let error = parse_error(&responses[0]);
15497            assert_eq!(error["code"], "version_unsupported");
15498            assert!(registry.get_module("aft").unwrap().is_none());
15499            assert_eq!(registry.active_registration_count().unwrap(), 0);
15500        }
15501    }
15502
15503    #[test]
15504    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15505        let registry = Arc::new(Registry::default());
15506        let forwarding = Arc::new(ForwardingTable::default());
15507        let handler =
15508            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15509        let module_connection = ConnectionId::new(301);
15510        let registration = registry
15511            .register_with_control_ops(
15512                manifest("aft-push", PROTOCOL_VERSION),
15513                PROTOCOL_VERSION,
15514                module_connection,
15515                module_baseline_control_ops(),
15516            )
15517            .unwrap();
15518        let (module_tx, _module_rx) = mpsc::channel(8);
15519        let endpoint = forwarding
15520            .register_module_connection(
15521                module_connection,
15522                "aft-push".to_string(),
15523                PROTOCOL_VERSION,
15524                manifest_concurrency(&registration.manifest),
15525                FrameSink::new(module_tx),
15526            )
15527            .unwrap();
15528
15529        // A push op this version does not know is ignored (forward-compat), not errored.
15530        let unknown = Frame::build(
15531            FrameType::Push,
15532            control_flags(),
15533            0,
15534            0,
15535            5,
15536            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15537        )
15538        .unwrap();
15539        let out = handler.handle_status_update(endpoint, unknown).unwrap();
15540        assert!(
15541            out.is_empty(),
15542            "unknown push op must be ignored, got {out:?}"
15543        );
15544
15545        // A malformed body for a KNOWN op is a real error worth surfacing.
15546        let malformed = Frame::build(
15547            FrameType::Push,
15548            control_flags(),
15549            0,
15550            0,
15551            6,
15552            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15553        )
15554        .unwrap();
15555        let out = handler.handle_status_update(endpoint, malformed).unwrap();
15556        assert_eq!(out.len(), 1);
15557        assert_eq!(out[0].header.ty, FrameType::Error);
15558        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15559    }
15560
15561    #[test]
15562    fn hello_rejected_when_connection_already_owns_client_routes() {
15563        let registry = Arc::new(Registry::default());
15564        let forwarding = Arc::new(ForwardingTable::default());
15565        let handler =
15566            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15567        // Commits a client route on connection 202 (bound to a module on conn 101).
15568        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
15569        let client_connection = ConnectionId::new(202);
15570
15571        // That same connection now tries to register as a module: rejected, so one
15572        // connection never holds both client-route and module-endpoint state.
15573        let responses = handler
15574            .handle_control(
15575                client_connection,
15576                hello_frame("aft-second", PROTOCOL_VERSION, 9),
15577            )
15578            .unwrap();
15579        assert_eq!(responses[0].header.ty, FrameType::Error);
15580        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15581        assert!(registry.get_module("aft-second").unwrap().is_none());
15582    }
15583
15584    #[tokio::test]
15585    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15586        let registry = Arc::new(Registry::default());
15587        let forwarding = Arc::new(ForwardingTable::default());
15588        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15589        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15590        hello_via_sink(
15591            &handler,
15592            &module_ctx,
15593            &mut module_rx,
15594            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15595        )
15596        .await;
15597        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15598        let pending = forwarding
15599            .begin_route_bind_relay_for_test(
15600                client_ctx.connection_id,
15601                client_ctx.egress.clone(),
15602                2,
15603                "alpha",
15604            )
15605            .unwrap();
15606        forwarding
15607            .complete_pending_relay(
15608                module_ctx.connection_id,
15609                pending.corr,
15610                RouteBindRelayOutcome::Accepted,
15611            )
15612            .unwrap();
15613        client_rx.try_recv().unwrap();
15614        for module_id in ["beta", "alpha"] {
15615            let replies = handler
15616                .handle_control_frame(
15617                    &module_ctx,
15618                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15619                )
15620                .await
15621                .unwrap();
15622            assert_eq!(replies.len(), 1, "second HELLO must be refused");
15623            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15624        }
15625        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15626        assert!(registry.get_module("beta").unwrap().is_none());
15627        assert!(matches!(
15628            forwarding
15629                .lookup_data_route(
15630                    client_ctx.connection_id,
15631                    pending.client_channel,
15632                    pending.client_epoch,
15633                )
15634                .unwrap(),
15635            DataRoute::Client(DataRouteState::Bound(_))
15636        ));
15637        assert!(handler
15638            .hello_launch_nonces
15639            .lock()
15640            .unwrap()
15641            .presented(module_ctx.connection_id, Some("alpha-nonce")));
15642        assert!(module_rx.try_recv().is_err());
15643    }
15644
15645    #[test]
15646    fn reserved_module_hello_requires_matching_launch_nonce() {
15647        let registry = Arc::new(Registry::default());
15648        let supervisor = SupervisorHandle::new();
15649        // The supervisor recorded the nonce it injected when it spawned the reserved
15650        // module; the HELLO verifier checks against the same shared handle.
15651        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15652        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15653
15654        // A HELLO with NO nonce is rejected.
15655        let no_nonce = handler
15656            .handle_control(
15657                ConnectionId::new(1),
15658                hello_frame("vault", PROTOCOL_VERSION, 1),
15659            )
15660            .unwrap();
15661        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15662        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15663        assert!(registry.get_module("vault").unwrap().is_none());
15664
15665        // A HELLO with the WRONG nonce is rejected.
15666        let wrong = handler
15667            .handle_control(
15668                ConnectionId::new(2),
15669                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15670            )
15671            .unwrap();
15672        assert_eq!(wrong[0].header.ty, FrameType::Error);
15673        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15674        assert!(registry.get_module("vault").unwrap().is_none());
15675
15676        // A HELLO with the CORRECT nonce registers.
15677        let ok = handler
15678            .handle_control(
15679                ConnectionId::new(3),
15680                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15681            )
15682            .unwrap();
15683        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15684        assert!(registry.get_module("vault").unwrap().is_some());
15685    }
15686
15687    #[test]
15688    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15689        let registry = Arc::new(Registry::default());
15690        let supervisor = SupervisorHandle::new();
15691        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15692        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15693        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15694
15695        let squat = handler
15696            .handle_control(
15697                ConnectionId::new(1),
15698                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15699            )
15700            .unwrap();
15701        assert_eq!(squat[0].header.ty, FrameType::Error);
15702        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15703        assert!(parse_error(&squat[0])["message"]
15704            .as_str()
15705            .unwrap()
15706            .contains("fed:"));
15707
15708        let accepted_peer = handler
15709            .handle_control(
15710                ConnectionId::new(2),
15711                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15712            )
15713            .unwrap();
15714        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15715
15716        let accepted_short = handler
15717            .handle_control(
15718                ConnectionId::new(3),
15719                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15720            )
15721            .unwrap();
15722        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15723
15724        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15725            let response = handler
15726                .handle_control(
15727                    ConnectionId::new(conn),
15728                    hello_frame(module_id, PROTOCOL_VERSION, conn),
15729                )
15730                .unwrap();
15731            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15732        }
15733    }
15734
15735    #[test]
15736    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15737        let registry = Arc::new(Registry::default());
15738        let supervisor = SupervisorHandle::new();
15739        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15740        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15741        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15742        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15743
15744        let owner_nonce = handler
15745            .handle_control(
15746                ConnectionId::new(1),
15747                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15748            )
15749            .unwrap();
15750        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15751        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15752        assert!(registry.get_module("fed:special").unwrap().is_none());
15753
15754        let exact_nonce = handler
15755            .handle_control(
15756                ConnectionId::new(2),
15757                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15758            )
15759            .unwrap();
15760        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15761        assert!(registry.get_module("fed:special").unwrap().is_some());
15762    }
15763
15764    #[test]
15765    fn non_reserved_module_ignores_launch_nonce() {
15766        let registry = Arc::new(Registry::default());
15767        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
15768        // registration succeeds whether a spawned process echoes a nonce or not.
15769        let handler = ControlHandler::new(Arc::clone(&registry));
15770        let no_nonce = handler
15771            .handle_control(
15772                ConnectionId::new(1),
15773                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15774            )
15775            .unwrap();
15776        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15777        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15778
15779        let echoed_nonce = handler
15780            .handle_control(
15781                ConnectionId::new(2),
15782                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15783            )
15784            .unwrap();
15785        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15786        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15787    }
15788
15789    #[test]
15790    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15791        let handler = ControlHandler::default();
15792        let conn = ConnectionId::new(1);
15793        let malformed = Frame::build(
15794            FrameType::Hello,
15795            control_flags(),
15796            0,
15797            0,
15798            3,
15799            b"{not json".to_vec(),
15800        )
15801        .unwrap();
15802
15803        let error = handler.handle_control(conn, malformed).unwrap();
15804        assert_eq!(error[0].header.ty, FrameType::Error);
15805        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15806
15807        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15808        let pong = handler.handle_control(conn, ping).unwrap();
15809        assert_eq!(pong[0].header.ty, FrameType::Pong);
15810        assert_eq!(pong[0].header.corr, 4);
15811    }
15812
15813    #[test]
15814    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15815        let registry = Arc::new(Registry::default());
15816        let handler = ControlHandler::new(Arc::clone(&registry));
15817
15818        handler
15819            .handle_control(
15820                ConnectionId::new(1),
15821                hello_frame("aft", PROTOCOL_VERSION, 1),
15822            )
15823            .unwrap();
15824        let duplicate = handler
15825            .handle_control(
15826                ConnectionId::new(2),
15827                hello_frame("aft", PROTOCOL_VERSION, 2),
15828            )
15829            .unwrap();
15830
15831        assert_eq!(duplicate[0].header.ty, FrameType::Error);
15832        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15833        let registration = registry.get_module("aft").unwrap().unwrap();
15834        assert_eq!(registration.connection_id, ConnectionId::new(1));
15835    }
15836
15837    #[test]
15838    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15839        let registry = Arc::new(Registry::default());
15840        let forwarding = Arc::new(ForwardingTable::default());
15841        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15842        let handler =
15843            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15844                .with_process_liveness(process_liveness);
15845        let (ctx, route_channel, route_epoch) =
15846            bind_liveness_route(&registry, &forwarding, "aft-dead");
15847        let responses = handler
15848            .handle_route_poll(
15849                &ctx,
15850                route_poll_frame(41, PollKind::Liveness, route_channel),
15851                route_channel,
15852                route_epoch,
15853                PollKind::Liveness,
15854            )
15855            .unwrap();
15856
15857        assert_eq!(responses.len(), 1);
15858        assert_eq!(responses[0].header.ty, FrameType::Response);
15859        assert_route_poll_liveness(&responses[0], false);
15860    }
15861
15862    #[test]
15863    fn liveness_poll_without_process_source_uses_bound_route() {
15864        let registry = Arc::new(Registry::default());
15865        let forwarding = Arc::new(ForwardingTable::default());
15866        let handler =
15867            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15868        let (ctx, route_channel, route_epoch) =
15869            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
15870        let responses = handler
15871            .handle_route_poll(
15872                &ctx,
15873                route_poll_frame(42, PollKind::Liveness, route_channel),
15874                route_channel,
15875                route_epoch,
15876                PollKind::Liveness,
15877            )
15878            .unwrap();
15879
15880        assert_route_poll_liveness(&responses[0], true);
15881    }
15882
15883    #[test]
15884    fn liveness_poll_untracked_process_source_uses_bound_route() {
15885        let registry = Arc::new(Registry::default());
15886        let forwarding = Arc::new(ForwardingTable::default());
15887        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15888        let handler =
15889            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15890                .with_process_liveness(process_liveness);
15891        let (ctx, route_channel, route_epoch) =
15892            bind_liveness_route(&registry, &forwarding, "aft-untracked");
15893        let responses = handler
15894            .handle_route_poll(
15895                &ctx,
15896                route_poll_frame(43, PollKind::Liveness, route_channel),
15897                route_channel,
15898                route_epoch,
15899                PollKind::Liveness,
15900            )
15901            .unwrap();
15902
15903        assert_route_poll_liveness(&responses[0], true);
15904    }
15905
15906    #[tokio::test]
15907    async fn unknown_op_returns_unknown_control_op() {
15908        let handler = ControlHandler::default();
15909        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15910        let request = Frame::build(
15911            FrameType::Request,
15912            control_flags(),
15913            0,
15914            0,
15915            55,
15916            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15917        )
15918        .unwrap();
15919
15920        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15921
15922        assert_eq!(response.len(), 1);
15923        assert_eq!(response[0].header.ty, FrameType::Error);
15924        assert_eq!(response[0].header.corr, 55);
15925        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15926    }
15927
15928    #[tokio::test]
15929    async fn supervisor_provenance_rejects_unknown_exact_module() {
15930        let handler = ControlHandler::default();
15931        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15932        let request = Frame::build(
15933            FrameType::Request,
15934            control_flags(),
15935            0,
15936            0,
15937            57,
15938            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15939        )
15940        .unwrap();
15941
15942        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15943
15944        assert_eq!(response.len(), 1);
15945        assert_eq!(response[0].header.ty, FrameType::Error);
15946        assert_eq!(response[0].header.corr, 57);
15947        let error = parse_error(&response[0]);
15948        assert_eq!(error["code"], "unknown_module");
15949        assert_eq!(error["message"], "module_id 'missing' is not supervised");
15950    }
15951
15952    #[test]
15953    fn provenance_probe_override_keeps_handler_tests_deterministic() {
15954        let expected = subc_control::RunningImageAgreement::Unavailable {
15955            reason: subc_control::RunningImageUnavailableReason::HashFailed,
15956        };
15957        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15958        assert_eq!(handler.provenance_probe_override, Some(expected));
15959    }
15960
15961    #[test]
15962    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15963        let verdict = reload_verdict(
15964            std::path::Path::new("/bin/new"),
15965            Some(std::path::Path::new("/bin/old")),
15966            subc_control::RunningImageAgreement::Unavailable {
15967                reason: subc_control::RunningImageUnavailableReason::HashFailed,
15968            },
15969        );
15970        assert!(matches!(
15971            verdict.path,
15972            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15973                if configured == std::path::Path::new("/bin/new")
15974                    && spawned_from == std::path::Path::new("/bin/old")
15975        ));
15976    }
15977
15978    #[test]
15979    fn launch_spec_without_running_digest_is_unavailable_not_match() {
15980        use subc_control::{
15981            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15982        };
15983        let configured = crate::supervise::ModuleSpec {
15984            carry_containment: false,
15985            module_id: "legacy".into(),
15986            program: "/module".into(),
15987            args: Vec::new(),
15988            env: Vec::new(),
15989            reserved: false,
15990            reserved_prefixes: Vec::new(),
15991            protocol: subc_control::ModuleProtocol::Subc,
15992            overlap: Default::default(),
15993        };
15994        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15995        assert_eq!(
15996            launch_spec_agreement(&configured, false, Some(&digest)),
15997            Agreement::Unavailable {
15998                reason: Reason::NotRunning
15999            }
16000        );
16001        assert_eq!(
16002            launch_spec_agreement(&configured, true, None),
16003            Agreement::Unavailable {
16004                reason: Reason::SpawnedSpecUnavailable
16005            }
16006        );
16007        assert_eq!(
16008            launch_spec_agreement(&configured, true, Some(&digest)),
16009            Agreement::Match
16010        );
16011    }
16012
16013    #[cfg(windows)]
16014    #[test]
16015    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16016        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16017        let configured = dir.join("module.exe");
16018        std::fs::write(&configured, b"executable path fixture").unwrap();
16019        let spawned = std::fs::canonicalize(&configured).unwrap();
16020        assert_ne!(
16021            configured, spawned,
16022            "fixture must exercise different path spellings"
16023        );
16024        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16025        let lower_case = configured.to_string_lossy().to_lowercase();
16026        for spelling in [
16027            &configured,
16028            Path::new(&forward_slashes),
16029            Path::new(&lower_case),
16030        ] {
16031            let verdict = reload_verdict(
16032                spelling,
16033                Some(&spawned),
16034                subc_control::RunningImageAgreement::Unavailable {
16035                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16036                },
16037            );
16038            assert_eq!(
16039                verdict.path,
16040                ReloadPathAgreement::Match,
16041                "configured {spelling:?}, spawned {spawned:?}"
16042            );
16043        }
16044    }
16045
16046    #[cfg(windows)]
16047    #[test]
16048    fn windows_reload_path_keeps_different_existing_program_pending() {
16049        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16050        let old = dir.join("old.exe");
16051        let new = dir.join("new.exe");
16052        std::fs::write(&old, b"same executable content").unwrap();
16053        std::fs::copy(&old, &new).unwrap();
16054        let spawned = std::fs::canonicalize(&old).unwrap();
16055        let image = subc_control::RunningImageAgreement::Unavailable {
16056            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16057        };
16058        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16059        assert!(
16060            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16061            if configured == new && spawned_from == spawned)
16062        );
16063        let missing = dir.join("missing.exe");
16064        assert!(matches!(
16065            reload_verdict(&missing, Some(&spawned), image).path,
16066            ReloadPathAgreement::Mismatch { .. }
16067        ));
16068    }
16069
16070    #[test]
16071    fn reload_verdict_detects_replaced_image_at_same_path() {
16072        let image = subc_control::RunningImageAgreement::Mismatch {
16073            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16074                digest: "old".into(),
16075            },
16076            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16077                digest: "new".into(),
16078            },
16079        };
16080        let verdict = reload_verdict(
16081            std::path::Path::new("/bin/same"),
16082            Some(std::path::Path::new("/bin/same")),
16083            image.clone(),
16084        );
16085        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16086        assert_eq!(verdict.image, image);
16087    }
16088
16089    #[test]
16090    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16091        let image = subc_control::RunningImageAgreement::Unavailable {
16092            reason: subc_control::RunningImageUnavailableReason::NotRunning,
16093        };
16094        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16095        assert_eq!(
16096            verdict.path,
16097            subc_control::ReloadPathAgreement::Unavailable {
16098                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16099            }
16100        );
16101        assert_eq!(verdict.image, image);
16102
16103        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16104            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16105        };
16106        let verdict = reload_verdict(
16107            std::path::Path::new("/bin/same"),
16108            Some(std::path::Path::new("/bin/same")),
16109            unconfirmed.clone(),
16110        );
16111        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16112        assert_eq!(verdict.image, unconfirmed);
16113    }
16114
16115    #[test]
16116    fn reload_verdict_preserves_each_image_unavailability_reason() {
16117        use subc_control::RunningImageUnavailableReason as Reason;
16118
16119        for reason in [
16120            Reason::NotRunning,
16121            Reason::UnsupportedPlatform,
16122            Reason::RunningExecutableUnreadable,
16123            Reason::SpawnedPathUnreadable,
16124            Reason::HashFailed,
16125            Reason::ProcessIdentityUnconfirmed,
16126            Reason::Unknown("future_probe_reason".to_string()),
16127        ] {
16128            let image = subc_control::RunningImageAgreement::Unavailable {
16129                reason: reason.clone(),
16130            };
16131            let verdict = reload_verdict(
16132                std::path::Path::new("/bin/same"),
16133                Some(std::path::Path::new("/bin/same")),
16134                image.clone(),
16135            );
16136            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16137            assert_eq!(verdict.image, image, "{reason:?}");
16138        }
16139    }
16140
16141    #[tokio::test]
16142    async fn malformed_control_bodies_return_invalid_control_body() {
16143        let handler = ControlHandler::default();
16144        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16145
16146        for (corr, body) in [
16147            (56, br#"{"route_channel":1}"#.as_slice()),
16148            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16149            (
16150                58,
16151                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16152            ),
16153        ] {
16154            let request = Frame::build(
16155                FrameType::Request,
16156                control_flags(),
16157                0,
16158                0,
16159                corr,
16160                body.to_vec(),
16161            )
16162            .unwrap();
16163            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16164
16165            assert_eq!(response.len(), 1);
16166            assert_eq!(response[0].header.ty, FrameType::Error);
16167            assert_eq!(response[0].header.corr, corr);
16168            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16169        }
16170    }
16171
16172    #[tokio::test]
16173    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16174        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16175        let registry = Arc::new(Registry::default());
16176        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16177        let router = Router::with_control_handler(Arc::clone(&control));
16178        let connection = router.begin_connection();
16179        let (ctx, mut rx) = route_ctx(connection.id());
16180
16181        router
16182            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16183            .await
16184            .unwrap();
16185        let response = rx.recv().await.unwrap();
16186        let ack = parse_ack(&response);
16187        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16188        let channel = 1;
16189
16190        let goodbye =
16191            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16192        router.route_for_connection(&ctx, goodbye).await.unwrap();
16193        assert!(rx.try_recv().is_err());
16194        assert!(registry.get_module("aft").unwrap().is_none());
16195
16196        router
16197            .route_for_connection(&ctx, channel_request(channel, 13))
16198            .await
16199            .unwrap();
16200        let error_frame = rx.recv().await.unwrap();
16201        assert_eq!(error_frame.header.ty, FrameType::Error);
16202        assert_eq!(error_frame.header.channel, channel);
16203        let captured = crate::router::test_log::captured_logs(&logs);
16204        assert_eq!(
16205            captured
16206                .lines()
16207                .filter(|line| {
16208                    line.contains("module_id=aft")
16209                        && line.contains("reason=explicit_goodbye")
16210                        && line.contains("module registration ended")
16211                })
16212                .count(),
16213            1,
16214            "unexpected GOODBYE registry log: {captured}"
16215        );
16216    }
16217
16218    #[tokio::test]
16219    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16220        let registry = Arc::new(Registry::default());
16221        let handler = ControlHandler::new(registry).with_capability_config(
16222            [("prov".to_string(), true), ("cons".to_string(), true)],
16223            BTreeMap::new(),
16224        );
16225        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16226        register_capability_manifest(
16227            &handler,
16228            &provider_ctx,
16229            &mut provider_rx,
16230            capability_manifest("prov", &["thing/v1"], &[]),
16231            1,
16232        )
16233        .await;
16234        let mut consumer = capability_manifest("cons", &[], &[]);
16235        consumer.capabilities.as_mut().unwrap().requires.push(
16236            subc_protocol::manifest::CapabilityRequirement {
16237                capability: "thing/v1".to_string(),
16238                need: subc_protocol::manifest::CapabilityNeed::Required,
16239            },
16240        );
16241        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16242        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16243        assert_eq!(
16244            handler.capability_evaluator.verdict("cons", "thing/v1"),
16245            Some(CapabilityVerdict::Provided)
16246        );
16247        let (mut client_rx, _) = open_route_for_capability_test(
16248            &handler,
16249            &provider_ctx,
16250            &mut provider_rx,
16251            703,
16252            3,
16253            "prov",
16254            None,
16255        )
16256        .await;
16257        let goodbye =
16258            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16259        handler
16260            .handle_control_frame(&provider_ctx, goodbye)
16261            .await
16262            .unwrap();
16263        assert_eq!(
16264            handler.capability_evaluator.verdict("cons", "thing/v1"),
16265            Some(CapabilityVerdict::NeverProvided)
16266        );
16267        let closed = client_rx
16268            .try_recv()
16269            .expect("GOODBYE pushes route.closed before route GOODBYE");
16270        assert!(
16271            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16272            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16273        );
16274        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16275        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16276    }
16277
16278    struct RouteClosedFixture {
16279        handler: ControlHandler,
16280        #[cfg(unix)]
16281        supervisor: Supervisor,
16282        module_ctx: RouteCtx,
16283        #[cfg(unix)]
16284        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16285        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16286        route: Arc<crate::forwarding::RouteBinding>,
16287    }
16288
16289    async fn route_closed_fixture() -> RouteClosedFixture {
16290        let registry = Arc::new(Registry::default());
16291        let handler = ControlHandler::new(Arc::clone(&registry));
16292        #[cfg(unix)]
16293        let supervisor = Supervisor::new(registry, RestartPolicy::default())
16294            .with_forwarding(Arc::clone(&handler.forwarding));
16295        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16296        register_capability_manifest(
16297            &handler,
16298            &module_ctx,
16299            &mut module_rx,
16300            capability_manifest("prov", &[], &[]),
16301            1,
16302        )
16303        .await;
16304        let (client_rx, bind) = open_route_for_capability_test(
16305            &handler,
16306            &module_ctx,
16307            &mut module_rx,
16308            702,
16309            2,
16310            "prov",
16311            None,
16312        )
16313        .await;
16314        let ModuleControlRequest::RouteBind {
16315            route_channel: channel,
16316            epoch: route_epoch,
16317            ..
16318        } = bind
16319        else {
16320            panic!("expected route.bind");
16321        };
16322        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16323            handler
16324                .forwarding
16325                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16326                .unwrap()
16327        else {
16328            panic!("the acknowledged route must be bound");
16329        };
16330        RouteClosedFixture {
16331            handler,
16332            #[cfg(unix)]
16333            supervisor,
16334            module_ctx,
16335            #[cfg(unix)]
16336            module_rx,
16337            client_rx,
16338            route,
16339        }
16340    }
16341
16342    impl RouteClosedFixture {
16343        #[cfg(unix)]
16344        async fn cut(&mut self) {
16345            let notices = async {
16346                let mut draining = self.module_rx.recv().await.unwrap();
16347                assert!(matches!(
16348                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16349                        &draining.body
16350                    )
16351                    .unwrap(),
16352                    subc_protocol::session::ModuleControlCommand::Draining {
16353                        reason: RouteCloseReason::Disable,
16354                        ..
16355                    }
16356                ));
16357                draining.flushed.take().unwrap().send(()).unwrap();
16358                let mut closing = self.client_rx.recv().await.unwrap();
16359                assert!(matches!(
16360                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16361                    ClientControlPush::RouteClosing {
16362                        reason: RouteCloseReason::Restart,
16363                        ..
16364                    }
16365                ));
16366                closing.flushed.take().unwrap().send(()).unwrap();
16367            };
16368            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16369            result.unwrap();
16370        }
16371
16372        fn disconnect(&mut self) -> ClientControlPush {
16373            self.handler
16374                .cleanup_connection(self.module_ctx.connection_id)
16375                .unwrap();
16376            let closed = self.client_rx.try_recv().unwrap();
16377            assert_eq!(closed.header.ty, FrameType::Push);
16378            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16379            assert!(matches!(
16380                &push,
16381                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16382                    if channels == &[self.route.client_channel]
16383            ));
16384            assert_eq!(
16385                self.client_rx.try_recv().unwrap().header.ty,
16386                FrameType::Goodbye
16387            );
16388            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16389            push
16390        }
16391    }
16392
16393    #[cfg(unix)]
16394    #[tokio::test(start_paused = true)]
16395    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16396        let mut fixture = route_closed_fixture().await;
16397        fixture.cut().await;
16398        assert!(matches!(
16399            fixture.disconnect(),
16400            ClientControlPush::RouteClosed {
16401                reason: RouteCloseReason::Restart,
16402                drained: true,
16403                ..
16404            }
16405        ));
16406    }
16407
16408    #[cfg(unix)]
16409    #[tokio::test(start_paused = true)]
16410    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16411        let mut fixture = route_closed_fixture().await;
16412        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16413        let started = tokio::time::Instant::now();
16414        fixture.cut().await;
16415        assert!(started.elapsed() >= Duration::from_secs(2));
16416        assert_eq!(fixture.route.flow.in_flight(), 1);
16417        assert!(matches!(
16418            fixture.disconnect(),
16419            ClientControlPush::RouteClosed {
16420                reason: RouteCloseReason::Restart,
16421                drained: false,
16422                ..
16423            }
16424        ));
16425    }
16426
16427    #[tokio::test]
16428    async fn module_connection_crash_reports_crash_and_not_drained() {
16429        let mut fixture = route_closed_fixture().await;
16430        // The server calls cleanup_connection on module EOF. Without a planned
16431        // daemon shutdown, that disconnect must remain a crash with no prior
16432        // route.closing notice, even when the route has no in-flight calls.
16433        assert!(matches!(
16434            fixture.disconnect(),
16435            ClientControlPush::RouteClosed {
16436                reason: RouteCloseReason::Crash,
16437                drained: false,
16438                ..
16439            }
16440        ));
16441    }
16442
16443    #[tokio::test]
16444    async fn dropping_router_connection_releases_registration() {
16445        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16446        let registry = Arc::new(Registry::default());
16447        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16448        let router = Router::with_control_handler(Arc::clone(&control));
16449        let connection = router.begin_connection();
16450        let connection_id = connection.id();
16451        let (ctx, mut rx) = route_ctx(connection_id);
16452
16453        router
16454            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16455            .await
16456            .unwrap();
16457        let response = rx.recv().await.unwrap();
16458        let ack = parse_ack(&response);
16459        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16460        assert!(registry.get_module("aft").unwrap().is_some());
16461
16462        drop(connection);
16463
16464        assert!(registry.get_module("aft").unwrap().is_none());
16465        assert_eq!(registry.active_registration_count().unwrap(), 0);
16466
16467        control
16468            .cleanup_connection(ConnectionId::new(u64::MAX))
16469            .unwrap();
16470        let captured = crate::router::test_log::captured_logs(&logs);
16471        println!("captured registry lifecycle logs:\n{captured}");
16472        let events: Vec<_> = captured
16473            .lines()
16474            .filter(|line| {
16475                line.contains("module registered module_id=aft ")
16476                    || line.contains("module registration ended")
16477            })
16478            .collect();
16479        assert_eq!(
16480            events.len(),
16481            2,
16482            "unexpected registry lifecycle logs: {captured}"
16483        );
16484        assert!(events[0].contains("module registered module_id=aft "));
16485        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16486        assert!(events[1].contains(&format!(
16487            "module_id=aft connection_id={}",
16488            connection_id.get()
16489        )));
16490        assert!(events[1].contains("reason=connection_closed"));
16491        assert!(events[1].contains("module registration ended"));
16492        assert_eq!(
16493            captured
16494                .lines()
16495                .filter(|line| line.contains("module registered module_id=aft "))
16496                .count(),
16497            1,
16498            "legacy registration admission line must appear once: {captured}"
16499        );
16500    }
16501
16502    #[tokio::test]
16503    async fn hello_registration_keeps_legacy_module_registered_line_once() {
16504        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16505        let registry = Arc::new(Registry::default());
16506        let control = ControlHandler::new(Arc::clone(&registry));
16507        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16508        hello_via_sink(
16509            &control,
16510            &ctx,
16511            &mut rx,
16512            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16513        )
16514        .await;
16515
16516        let captured = crate::router::test_log::captured_logs(&logs);
16517        let admissions: Vec<_> = captured
16518            .lines()
16519            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16520            .collect();
16521        assert_eq!(
16522            admissions.len(),
16523            1,
16524            "legacy admission line must remain exactly once: {captured}"
16525        );
16526        assert!(admissions[0].contains("routable_provider=true"));
16527        assert!(admissions[0].contains("connection_id=777"));
16528    }
16529
16530    fn capability_manifest(
16531        module_id: &str,
16532        provides: &[&str],
16533        must_never_reach: &[&str],
16534    ) -> ModuleManifest {
16535        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16536        manifest.capabilities = Some(CapabilityDeclarations {
16537            provides: provides
16538                .iter()
16539                .map(|capability| (*capability).to_string())
16540                .collect(),
16541            requires: Vec::new(),
16542            must_never_reach: must_never_reach
16543                .iter()
16544                .map(|capability| (*capability).to_string())
16545                .collect(),
16546        });
16547        manifest
16548    }
16549
16550    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16551        Frame::build(
16552            FrameType::Hello,
16553            control_flags(),
16554            0,
16555            0,
16556            corr,
16557            serde_json::to_vec(&ModuleHelloBody {
16558                protocol_ver: manifest.protocol_ver,
16559                manifest,
16560                control_ops: None,
16561                launch_nonce: None,
16562            })
16563            .expect("capability test HELLO serializes"),
16564        )
16565        .expect("capability test HELLO frame builds")
16566    }
16567
16568    fn catalog_update_with_capabilities_frame(
16569        corr: u64,
16570        capabilities: CapabilityDeclarations,
16571    ) -> Frame {
16572        Frame::build(
16573            FrameType::Request,
16574            control_flags(),
16575            0,
16576            0,
16577            corr,
16578            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16579                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16580                capabilities: Some(capabilities),
16581                ready: None,
16582            })
16583            .expect("capability catalog.update serializes"),
16584        )
16585        .expect("capability catalog.update frame builds")
16586    }
16587
16588    async fn register_capability_manifest(
16589        handler: &ControlHandler,
16590        ctx: &RouteCtx,
16591        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16592        manifest: ModuleManifest,
16593        corr: u64,
16594    ) {
16595        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16596    }
16597
16598    async fn open_route_for_capability_test(
16599        handler: &ControlHandler,
16600        target_ctx: &RouteCtx,
16601        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16602        client_connection_id: u64,
16603        corr: u64,
16604        target_module_id: &str,
16605        consumer_identity: Option<ConsumerIdentity>,
16606    ) -> (
16607        mpsc::Receiver<crate::router::OutboundFrame>,
16608        ModuleControlRequest,
16609    ) {
16610        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16611        let route_handler = handler.clone();
16612        let target_module_id = target_module_id.to_string();
16613        let route_task = tokio::spawn(async move {
16614            route_handler
16615                .handle_control_frame(
16616                    &client_ctx,
16617                    route_open_frame_with_admission_facts(
16618                        corr,
16619                        &target_module_id,
16620                        unique_project_root("admission-facts"),
16621                        consumer_identity,
16622                        None,
16623                    ),
16624                )
16625                .await
16626                .expect("capability test route.open succeeds")
16627        });
16628        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16629            .await
16630            .expect("capability test route.open must reach route.bind")
16631            .expect("target control receiver stays open");
16632        let bind_request: ModuleControlRequest =
16633            serde_json::from_slice(&bind.body).expect("route.bind decodes");
16634        handler
16635            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16636            .await
16637            .expect("capability test route.bind ACK succeeds");
16638        assert!(route_task.await.expect("route.open task joins").is_empty());
16639        let opened = client_rx
16640            .recv()
16641            .await
16642            .expect("successful route.open publishes a response");
16643        assert!(matches!(
16644            serde_json::from_slice::<ClientControlResponse>(&opened.body),
16645            Ok(ClientControlResponse::RouteOpen { .. })
16646        ));
16647        (client_rx, bind_request)
16648    }
16649
16650    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16651        assert_eq!(frame.header.ty, FrameType::Push);
16652        assert_eq!(frame.header.channel, 0);
16653        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16654            .expect("route.closed control push decodes");
16655        let ClientControlPush::RouteClosed { channels, .. } = &push else {
16656            panic!("expected route.closed");
16657        };
16658        assert_eq!(channels.len(), 1, "exactly one violating route closed");
16659        let closed_channel = channels[0];
16660        let channels = channels.clone();
16661        assert_eq!(
16662            push,
16663            ClientControlPush::RouteClosed {
16664                module_id: target_module_id.to_string(),
16665                channels,
16666                reason: RouteCloseReason::CapabilityDenied,
16667                drained: false,
16668                abandoned: 0,
16669                excluded_subscriptions: 0,
16670                terminal: Some(false),
16671            }
16672        );
16673        closed_channel
16674    }
16675
16676    #[tokio::test]
16677    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16678        let registry = Arc::new(Registry::default());
16679        let forwarding = Arc::new(ForwardingTable::default());
16680        let supervisor = SupervisorHandle::new();
16681        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16682        let handler =
16683            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16684                .with_supervisor(supervisor);
16685        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16686        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16687        register_capability_manifest(
16688            &handler,
16689            &target_ctx,
16690            &mut target_rx,
16691            capability_manifest("target", &["credentials-provider/v1"], &[]),
16692            1,
16693        )
16694        .await;
16695        register_capability_manifest(
16696            &handler,
16697            &opener_ctx,
16698            &mut opener_rx,
16699            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16700            2,
16701        )
16702        .await;
16703
16704        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16705        let replies = handler
16706            .handle_control_frame(
16707                &client_ctx,
16708                route_open_frame_with_admission_facts(
16709                    3,
16710                    "target",
16711                    unique_project_root("admission-facts"),
16712                    Some(ConsumerIdentity {
16713                        module_id: "opener".to_string(),
16714                        launch_nonce: "opener-nonce".to_string(),
16715                    }),
16716                    None,
16717                ),
16718            )
16719            .await
16720            .expect("denied route.open returns a typed frame");
16721        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16722        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16723        assert!(
16724            target_rx.try_recv().is_err(),
16725            "forbidden route.open must not relay route.bind"
16726        );
16727    }
16728
16729    #[tokio::test]
16730    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16731        let registry = Arc::new(Registry::default());
16732        let forwarding = Arc::new(ForwardingTable::default());
16733        let supervisor = SupervisorHandle::new();
16734        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16735        let handler =
16736            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16737                .with_supervisor(supervisor);
16738        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16739        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16740        register_capability_manifest(
16741            &handler,
16742            &target_ctx,
16743            &mut target_rx,
16744            capability_manifest("target", &["credentials-provider/v1"], &[]),
16745            1,
16746        )
16747        .await;
16748        register_capability_manifest(
16749            &handler,
16750            &old_opener_ctx,
16751            &mut old_opener_rx,
16752            capability_manifest("opener", &[], &[]),
16753            2,
16754        )
16755        .await;
16756        let (mut client_rx, _) = open_route_for_capability_test(
16757            &handler,
16758            &target_ctx,
16759            &mut target_rx,
16760            712,
16761            3,
16762            "target",
16763            Some(ConsumerIdentity {
16764                module_id: "opener".to_string(),
16765                launch_nonce: "opener-nonce".to_string(),
16766            }),
16767        )
16768        .await;
16769        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16770
16771        handler
16772            .cleanup_connection(old_opener_ctx.connection_id)
16773            .expect("old opener registration cleans up");
16774        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16775        register_capability_manifest(
16776            &handler,
16777            &new_opener_ctx,
16778            &mut new_opener_rx,
16779            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16780            4,
16781        )
16782        .await;
16783
16784        let closed_channel = assert_capability_denied_push(
16785            client_rx
16786                .try_recv()
16787                .expect("HELLO deny addition must emit route.closed")
16788                .frame,
16789            "target",
16790        );
16791        let goodbye = client_rx
16792            .try_recv()
16793            .expect("HELLO deny addition must send the client a route GOODBYE")
16794            .frame;
16795        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16796        assert_eq!(goodbye.header.channel, closed_channel);
16797        assert_ne!(goodbye.header.channel, 0);
16798        assert_ne!(goodbye.header.epoch, 0);
16799        assert!(client_rx.try_recv().is_err());
16800        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16801        assert!(matches!(
16802            target_rx.try_recv(),
16803            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16804        ));
16805    }
16806
16807    #[tokio::test]
16808    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16809        let registry = Arc::new(Registry::default());
16810        let forwarding = Arc::new(ForwardingTable::default());
16811        let supervisor = SupervisorHandle::new();
16812        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16813        let handler =
16814            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16815                .with_supervisor(supervisor);
16816        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16817        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16818        register_capability_manifest(
16819            &handler,
16820            &target_ctx,
16821            &mut target_rx,
16822            capability_manifest("target", &[], &[]),
16823            1,
16824        )
16825        .await;
16826        register_capability_manifest(
16827            &handler,
16828            &opener_ctx,
16829            &mut opener_rx,
16830            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16831            2,
16832        )
16833        .await;
16834        let (mut client_rx, _) = open_route_for_capability_test(
16835            &handler,
16836            &target_ctx,
16837            &mut target_rx,
16838            722,
16839            3,
16840            "target",
16841            Some(ConsumerIdentity {
16842                module_id: "opener".to_string(),
16843                launch_nonce: "opener-nonce".to_string(),
16844            }),
16845        )
16846        .await;
16847        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16848
16849        let replies = handler
16850            .handle_control_frame(
16851                &target_ctx,
16852                catalog_update_with_capabilities_frame(
16853                    4,
16854                    CapabilityDeclarations {
16855                        provides: vec!["credentials-provider/v1".to_string()],
16856                        requires: Vec::new(),
16857                        must_never_reach: Vec::new(),
16858                    },
16859                ),
16860            )
16861            .await
16862            .expect("claim catalog.update succeeds");
16863        assert!(matches!(
16864            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16865            Ok(ModuleControlResponseToModule::CatalogUpdate {})
16866        ));
16867        let closed_channel = assert_capability_denied_push(
16868            client_rx
16869                .try_recv()
16870                .expect("claim addition must emit route.closed")
16871                .frame,
16872            "target",
16873        );
16874        let goodbye = client_rx
16875            .try_recv()
16876            .expect("claim addition must send the client a route GOODBYE")
16877            .frame;
16878        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16879        assert_eq!(goodbye.header.channel, closed_channel);
16880        assert_ne!(goodbye.header.channel, 0);
16881        assert_ne!(goodbye.header.epoch, 0);
16882        assert!(client_rx.try_recv().is_err());
16883        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16884        assert!(matches!(
16885            target_rx.try_recv(),
16886            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16887        ));
16888    }
16889
16890    #[tokio::test]
16891    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16892        let registry = Arc::new(Registry::default());
16893        let forwarding = Arc::new(ForwardingTable::default());
16894        let supervisor = SupervisorHandle::new();
16895        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16896        let handler =
16897            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16898                .with_supervisor(supervisor);
16899        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16900        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16901        register_capability_manifest(
16902            &handler,
16903            &target_ctx,
16904            &mut target_rx,
16905            capability_manifest("target", &["credentials-provider/v1"], &[]),
16906            1,
16907        )
16908        .await;
16909        register_capability_manifest(
16910            &handler,
16911            &opener_ctx,
16912            &mut opener_rx,
16913            capability_manifest("opener", &[], &[]),
16914            2,
16915        )
16916        .await;
16917        let (mut client_rx, _) = open_route_for_capability_test(
16918            &handler,
16919            &target_ctx,
16920            &mut target_rx,
16921            732,
16922            3,
16923            "target",
16924            Some(ConsumerIdentity {
16925                module_id: "opener".to_string(),
16926                launch_nonce: "opener-nonce".to_string(),
16927            }),
16928        )
16929        .await;
16930        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16931
16932        handler
16933            .handle_control_frame(
16934                &target_ctx,
16935                catalog_update_with_capabilities_frame(
16936                    4,
16937                    CapabilityDeclarations {
16938                        provides: Vec::new(),
16939                        requires: Vec::new(),
16940                        must_never_reach: Vec::new(),
16941                    },
16942                ),
16943            )
16944            .await
16945            .expect("claim removal catalog.update succeeds");
16946        assert_eq!(
16947            forwarding.active_binding_count().unwrap(),
16948            1,
16949            "removing an attested target claim must leave the route census unchanged"
16950        );
16951        assert!(
16952            client_rx.try_recv().is_err(),
16953            "claim removal must not emit route.closed or a client route GOODBYE"
16954        );
16955        assert!(
16956            target_rx.try_recv().is_err(),
16957            "claim removal must not send the target a route GOODBYE"
16958        );
16959    }
16960
16961    /// A direct client may open a route to a denied capability provider; this
16962    /// policy applies only to attested supervised module origins, not to direct clients.
16963    #[tokio::test]
16964    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16965        let registry = Arc::new(Registry::default());
16966        let forwarding = Arc::new(ForwardingTable::default());
16967        let supervisor = SupervisorHandle::new();
16968        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16969        let handler =
16970            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16971                .with_supervisor(supervisor);
16972        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16973        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16974        register_capability_manifest(
16975            &handler,
16976            &target_ctx,
16977            &mut target_rx,
16978            capability_manifest("target", &["credentials-provider/v1"], &[]),
16979            1,
16980        )
16981        .await;
16982        register_capability_manifest(
16983            &handler,
16984            &opener_ctx,
16985            &mut opener_rx,
16986            capability_manifest("opener", &[], &["credentials-provider/v1"]),
16987            2,
16988        )
16989        .await;
16990
16991        let (_client_rx, bind) = open_route_for_capability_test(
16992            &handler,
16993            &target_ctx,
16994            &mut target_rx,
16995            742,
16996            3,
16997            "target",
16998            None,
16999        )
17000        .await;
17001        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17002            panic!("direct scope-honesty route must bind");
17003        };
17004        assert_eq!(principal, Some(Principal::Direct));
17005        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17006    }
17007
17008    /// A module that denies a capability receives no self-route exemption when it
17009    /// also attestedly provides that capability.
17010    #[tokio::test]
17011    async fn must_never_reach_self_route_is_capability_forbidden() {
17012        let registry = Arc::new(Registry::default());
17013        let forwarding = Arc::new(ForwardingTable::default());
17014        let supervisor = SupervisorHandle::new();
17015        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17016        let handler =
17017            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17018                .with_supervisor(supervisor);
17019        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17020        register_capability_manifest(
17021            &handler,
17022            &self_ctx,
17023            &mut self_rx,
17024            capability_manifest(
17025                "self-provider",
17026                &["credentials-provider/v1"],
17027                &["credentials-provider/v1"],
17028            ),
17029            1,
17030        )
17031        .await;
17032
17033        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17034        let replies = handler
17035            .handle_control_frame(
17036                &client_ctx,
17037                route_open_frame_with_admission_facts(
17038                    2,
17039                    "self-provider",
17040                    unique_project_root("admission-facts"),
17041                    Some(ConsumerIdentity {
17042                        module_id: "self-provider".to_string(),
17043                        launch_nonce: "self-nonce".to_string(),
17044                    }),
17045                    None,
17046                ),
17047            )
17048            .await
17049            .expect("self-route refusal returns a typed frame");
17050        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17051        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17052        assert!(
17053            self_rx.try_recv().is_err(),
17054            "self denial must not relay route.bind"
17055        );
17056    }
17057
17058    #[test]
17059    fn unsupported_channel_zero_frame_returns_error() {
17060        let handler = ControlHandler::default();
17061        let request = Frame::build(
17062            FrameType::Request,
17063            control_flags(),
17064            0,
17065            0,
17066            21,
17067            b"opaque".to_vec(),
17068        )
17069        .unwrap();
17070
17071        let response = handler
17072            .handle_control(ConnectionId::new(1), request)
17073            .unwrap();
17074
17075        assert_eq!(response[0].header.ty, FrameType::Error);
17076        assert_eq!(
17077            parse_error(&response[0])["code"],
17078            "unsupported_control_frame"
17079        );
17080    }
17081
17082    /// Blue/green swap at the control-plane boundary. The supervisor that opens
17083    /// a swap is not wired yet, so the candidate is registered here directly
17084    /// into the registry and forwarding candidate slots, the way the swap's
17085    /// HELLO admission will.
17086    mod swap {
17087        use super::*;
17088
17089        const INCUMBENT: ConnectionId = ConnectionId::new(30);
17090        const CANDIDATE: ConnectionId = ConnectionId::new(40);
17091
17092        struct Swap {
17093            registry: Arc<Registry>,
17094            forwarding: Arc<ForwardingTable>,
17095            handler: ControlHandler,
17096            incumbent_ctx: RouteCtx,
17097            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17098            candidate_ctx: RouteCtx,
17099            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17100        }
17101
17102        async fn swap_with_incumbent() -> Swap {
17103            let registry = Arc::new(Registry::default());
17104            let forwarding = Arc::new(ForwardingTable::default());
17105            let handler =
17106                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17107            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17108            hello_via_sink(
17109                &handler,
17110                &incumbent_ctx,
17111                &mut incumbent_rx,
17112                hello_frame("aft", PROTOCOL_VERSION, 7),
17113            )
17114            .await;
17115            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17116            Swap {
17117                registry,
17118                forwarding,
17119                handler,
17120                incumbent_ctx,
17121                incumbent_rx,
17122                candidate_ctx,
17123                candidate_rx,
17124            }
17125        }
17126
17127        fn register_candidate(swap: &Swap, ready: Option<bool>) {
17128            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17129            candidate_manifest.ready = ready;
17130            let registration = swap
17131                .registry
17132                .register_candidate_with_control_ops(
17133                    candidate_manifest,
17134                    PROTOCOL_VERSION,
17135                    CANDIDATE,
17136                    module_baseline_control_ops(),
17137                )
17138                .unwrap();
17139            swap.forwarding
17140                .register_candidate_module_connection(
17141                    CANDIDATE,
17142                    "aft".to_string(),
17143                    PROTOCOL_VERSION,
17144                    manifest_concurrency(&registration.manifest),
17145                    swap.candidate_ctx.egress.clone(),
17146                )
17147                .unwrap();
17148        }
17149
17150        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17151            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17152            swap.registry.promote_candidate("aft").unwrap().unwrap();
17153            cutover.incumbent.unwrap()
17154        }
17155
17156        /// A swap candidate's HELLO reserves its candidate registry entry
17157        /// before installing its candidate forwarding endpoint. The swap
17158        /// supervisor promotes a ready candidate as soon as it reads it as
17159        /// registered, and promotion cuts forwarding over to the candidate
17160        /// endpoint, so the lookup the supervisor reads
17161        /// (`Registry::published_registration`) must not return the candidate
17162        /// until that endpoint exists. Otherwise the cutover finds no candidate
17163        /// endpoint and the swap fails as if the candidate had disconnected.
17164        #[tokio::test]
17165        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17166            let mut swap = swap_with_incumbent().await;
17167            let hold = Arc::new(std::sync::Barrier::new(2));
17168            let handler = swap
17169                .handler
17170                .clone()
17171                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17172            handler
17173                .supervisor
17174                .open_swap("aft", "candidate-nonce".to_string());
17175            let hello = {
17176                let handler = handler.clone();
17177                let candidate_ctx = swap.candidate_ctx.clone();
17178                std::thread::spawn(move || {
17179                    handler
17180                        .handle_hello(
17181                            candidate_ctx.connection_id,
17182                            Some(candidate_ctx.egress.clone()),
17183                            hello_frame_with_nonce(
17184                                "aft",
17185                                PROTOCOL_VERSION,
17186                                8,
17187                                Some("candidate-nonce"),
17188                            ),
17189                        )
17190                        .unwrap()
17191                })
17192            };
17193
17194            hold.wait();
17195            // Parked: the candidate slot is claimed but not published, and
17196            // forwarding has no candidate endpoint.
17197            assert!(swap
17198                .registry
17199                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17200                .unwrap()
17201                .is_some());
17202            assert!(swap
17203                .registry
17204                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17205                .unwrap()
17206                .is_none());
17207            assert!(swap
17208                .forwarding
17209                .module_endpoint_for_connection(CANDIDATE)
17210                .unwrap()
17211                .is_none());
17212
17213            hold.wait();
17214            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17215            assert_eq!(
17216                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17217                FrameType::HelloAck
17218            );
17219            let published = swap
17220                .registry
17221                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17222                .unwrap()
17223                .expect("candidate is published once its HELLO completes");
17224            assert_eq!(published.connection_id, CANDIDATE);
17225            assert!(swap
17226                .forwarding
17227                .module_endpoint_for_connection(CANDIDATE)
17228                .unwrap()
17229                .is_some());
17230        }
17231
17232        fn keyed_total(counters: &Value, key: &str) -> u64 {
17233            counters[key]
17234                .as_object()
17235                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17236                .unwrap_or(0)
17237        }
17238
17239        #[tokio::test]
17240        async fn replacement_logs_old_end_and_new_admission_once() {
17241            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17242            let mut swap = swap_with_incumbent().await;
17243            swap.handler
17244                .supervisor
17245                .open_swap("aft", "candidate-nonce".to_string());
17246            hello_via_sink(
17247                &swap.handler,
17248                &swap.candidate_ctx,
17249                &mut swap.candidate_rx,
17250                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17251            )
17252            .await;
17253            cutover(&swap);
17254            swap.handler.cleanup_connection(INCUMBENT).unwrap();
17255
17256            let captured = crate::router::test_log::captured_logs(&logs);
17257            println!("captured replacement registry logs:\n{captured}");
17258            let new_admissions: Vec<_> = captured
17259                .lines()
17260                .filter(|line| {
17261                    line.contains("connection_id=40")
17262                        && line.contains("swap candidate registered; not routable until cutover")
17263                })
17264                .collect();
17265            assert_eq!(
17266                new_admissions.len(),
17267                1,
17268                "unexpected admission logs: {captured}"
17269            );
17270            assert!(new_admissions[0].contains("module_id=aft"));
17271            assert!(new_admissions[0].contains("connection_id=40"));
17272            assert!(
17273                new_admissions[0].contains("swap candidate registered; not routable until cutover")
17274            );
17275            assert!(new_admissions[0].contains("ready=true"));
17276
17277            let old_admissions: Vec<_> = captured
17278                .lines()
17279                .filter(|line| {
17280                    line.contains("module registered module_id=aft ")
17281                        && line.contains("connection_id=30")
17282                })
17283                .collect();
17284            assert_eq!(
17285                old_admissions.len(),
17286                1,
17287                "unexpected incumbent admission: {captured}"
17288            );
17289
17290            let old_ends: Vec<_> = captured
17291                .lines()
17292                .filter(|line| {
17293                    line.contains("connection_id=30") && line.contains("module registration ended")
17294                })
17295                .collect();
17296            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17297            assert!(old_ends[0].contains("module_id=aft"));
17298            assert!(old_ends[0].contains("connection_id=30"));
17299            assert!(old_ends[0].contains("reason=replaced"));
17300            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17301            assert_eq!(
17302                captured
17303                    .lines()
17304                    .filter(|line| line.contains("module registration promoted"))
17305                    .count(),
17306                1,
17307                "unexpected promotion logs: {captured}"
17308            );
17309        }
17310
17311        /// An ack from the incumbent for a bind it was sent before cutover,
17312        /// arriving before the incumbent is drained. The incumbent is the live
17313        /// connection carrying every other client's routes, so the ack must
17314        /// not end it: the waiting client is told to retry, the reservation is
17315        /// given back, and the incumbent is told to drop just that binding.
17316        #[tokio::test]
17317        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17318            let mut swap = swap_with_incumbent().await;
17319            let handler = swap.handler.clone();
17320
17321            // A co-tenant route, bound on the incumbent before the swap.
17322            let cotenant = ConnectionId::new(31);
17323            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17324            let (cotenant_task, cotenant_bind) = relay_route_open(
17325                &handler,
17326                cotenant,
17327                &cotenant_ctx.egress,
17328                &mut swap.incumbent_rx,
17329                100,
17330                "aft",
17331                "swap-cotenant",
17332            )
17333            .await;
17334            handler
17335                .handle_control_frame(
17336                    &swap.incumbent_ctx,
17337                    route_bind_ack(cotenant_bind.header.corr),
17338                )
17339                .await
17340                .unwrap();
17341            assert!(cotenant_task.await.unwrap().is_empty());
17342            let (cotenant_channel, cotenant_epoch) =
17343                published_route(&cotenant_rx.recv().await.unwrap());
17344
17345            // A second route.open, relayed to the incumbent and not yet acked.
17346            let caller = ConnectionId::new(32);
17347            let (caller_ctx, mut caller_rx) = route_ctx(caller);
17348            let (caller_task, caller_bind) = relay_route_open(
17349                &handler,
17350                caller,
17351                &caller_ctx.egress,
17352                &mut swap.incumbent_rx,
17353                101,
17354                "aft",
17355                "swap-caller",
17356            )
17357            .await;
17358            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17359
17360            register_candidate(&swap, None);
17361            cutover(&swap);
17362
17363            // The incumbent acks after promotion and before any drain.
17364            let ack = handler
17365                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17366                .await;
17367            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17368            if module_loop_error.is_some() {
17369                // What the connection loop does with an untranslated router
17370                // error: end the connection, releasing every route on it.
17371                handler.cleanup_connection(INCUMBENT).unwrap();
17372            }
17373
17374            // 1. The incumbent's other routes survive.
17375            assert!(
17376                cotenant_rx.try_recv().is_err(),
17377                "the co-tenant route on the incumbent was torn down by one late ack: \
17378                 {module_loop_error:?}"
17379            );
17380            assert!(matches!(
17381                swap.forwarding
17382                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17383                    .unwrap(),
17384                DataRoute::Client(DataRouteState::Bound(_))
17385            ));
17386            assert_eq!(module_loop_error, None);
17387            assert!(swap
17388                .registry
17389                .get_module_by_connection(INCUMBENT)
17390                .unwrap()
17391                .is_some());
17392
17393            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
17394            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17395                .await
17396                .expect("the incumbent is told to drop the abandoned binding")
17397                .unwrap()
17398                .frame;
17399            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17400            assert_eq!(goodbye.header.channel, abandoned_channel);
17401            assert_eq!(goodbye.header.epoch, abandoned_epoch);
17402            assert!(swap.incumbent_rx.try_recv().is_err());
17403
17404            // 3. The waiting client gets a retryable refusal and no route.
17405            let response = caller_task.await.unwrap();
17406            assert_eq!(response.len(), 1);
17407            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17408            assert!(caller_rx.try_recv().is_err());
17409
17410            // 4. The reservation pair is given back, and the pending bind
17411            //    settled exactly once: one accepted open (the co-tenant) and one
17412            //    refused open (the caller), nothing counted twice.
17413            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17414            let counters = handler.counters().snapshot();
17415            assert_eq!(
17416                keyed_total(&counters, "route_open_accepted_by_principal"),
17417                1
17418            );
17419            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17420            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17421        }
17422
17423        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
17424        /// id would resolve to the promoted candidate and every new route.open
17425        /// would be refused as reloading, leaving neither process routable.
17426        #[tokio::test]
17427        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17428            let mut swap = swap_with_incumbent().await;
17429            register_candidate(&swap, None);
17430            let incumbent = cutover(&swap);
17431            swap.forwarding
17432                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17433                .unwrap()
17434                .expect("the incumbent is still registered");
17435
17436            let client = ConnectionId::new(33);
17437            let (client_ctx, mut client_rx) = route_ctx(client);
17438            let route_handler = swap.handler.clone();
17439            let open_ctx = RouteCtx {
17440                connection_id: client,
17441                egress: client_ctx.egress.clone(),
17442            };
17443            let mut route_task = tokio::spawn(async move {
17444                route_handler
17445                    .handle_control_frame(
17446                        &open_ctx,
17447                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17448                    )
17449                    .await
17450                    .unwrap()
17451            });
17452            let bind = tokio::select! {
17453                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17454                response = &mut route_task => {
17455                    let response = response.unwrap();
17456                    panic!(
17457                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
17458                        parse_error(&response[0])["code"]
17459                    );
17460                }
17461            };
17462            swap.handler
17463                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17464                .await
17465                .unwrap();
17466            assert!(route_task.await.unwrap().is_empty());
17467            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17468            match swap
17469                .forwarding
17470                .lookup_data_route(client, channel, epoch)
17471                .unwrap()
17472            {
17473                DataRoute::Client(DataRouteState::Bound(route)) => {
17474                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17475                }
17476                other => panic!("expected a bound route on the candidate, got {other:?}"),
17477            }
17478            assert!(swap.incumbent_rx.try_recv().is_err());
17479        }
17480
17481        /// A candidate declares itself ready with `catalog.update` on its own
17482        /// connection. If the connection-keyed registry lookups searched only the
17483        /// active slot, this would answer `not_registered` and the candidate
17484        /// would never become ready.
17485        #[tokio::test]
17486        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17487            let swap = swap_with_incumbent().await;
17488            register_candidate(&swap, Some(false));
17489            let update = Frame::build(
17490                FrameType::Request,
17491                control_flags(),
17492                0,
17493                0,
17494                55,
17495                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17496                    provides: manifest("aft", PROTOCOL_VERSION).provides,
17497                    capabilities: None,
17498                    ready: Some(true),
17499                })
17500                .unwrap(),
17501            )
17502            .unwrap();
17503
17504            let replies = swap
17505                .handler
17506                .handle_control_frame(&swap.candidate_ctx, update)
17507                .await
17508                .unwrap();
17509
17510            assert_eq!(replies.len(), 1);
17511            assert_eq!(
17512                replies[0].header.ty,
17513                FrameType::Response,
17514                "candidate catalog.update was refused: {:?}",
17515                serde_json::from_slice::<Value>(&replies[0].body).ok()
17516            );
17517            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17518            assert_eq!(
17519                swap.registry
17520                    .get_module("aft")
17521                    .unwrap()
17522                    .unwrap()
17523                    .connection_id,
17524                INCUMBENT
17525            );
17526        }
17527    }
17528
17529    /// The HELLO gate while the supervisor has a swap open: only the nonce it
17530    /// minted for the candidate admits a second process, into the candidate
17531    /// slot, and that check runs ahead of the reserved-module gate.
17532    mod swap_admission {
17533        use super::*;
17534
17535        const INCUMBENT_NONCE: &str = "incumbent-nonce";
17536        const CANDIDATE_NONCE: &str = "candidate-nonce";
17537
17538        fn handler_with_incumbent(
17539            module_id: &str,
17540            reserved: bool,
17541        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17542            let registry = Arc::new(Registry::default());
17543            let supervisor = SupervisorHandle::new();
17544            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17545            if reserved {
17546                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17547            }
17548            let handler =
17549                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
17550            let incumbent = handler
17551                .handle_control(
17552                    ConnectionId::new(1),
17553                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17554                )
17555                .unwrap();
17556            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17557            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17558            (registry, supervisor, handler)
17559        }
17560
17561        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
17562        /// admits every nonce, so while a swap is open the swap gate is the only
17563        /// thing between a key-holder and the candidate slot. A nonce the
17564        /// supervisor did not mint, or none at all, is refused, and neither the
17565        /// incumbent's registration nor the candidate slot moves.
17566        #[test]
17567        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17568            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17569
17570            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17571                let replies = handler
17572                    .handle_control(
17573                        ConnectionId::new(connection),
17574                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17575                    )
17576                    .unwrap();
17577                assert_eq!(replies[0].header.ty, FrameType::Error);
17578                assert_eq!(
17579                    parse_error(&replies[0])["code"],
17580                    "swap_token_invalid",
17581                    "nonce {nonce:?}"
17582                );
17583            }
17584            assert!(registry.get_candidate("aft").unwrap().is_none());
17585            assert_eq!(
17586                registry.get_module("aft").unwrap().unwrap().connection_id,
17587                ConnectionId::new(1)
17588            );
17589
17590            // Control: the minted token is admitted, into the candidate slot,
17591            // and only once.
17592            let admitted = handler
17593                .handle_control(
17594                    ConnectionId::new(4),
17595                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17596                )
17597                .unwrap();
17598            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17599            assert_eq!(
17600                registry
17601                    .get_candidate("aft")
17602                    .unwrap()
17603                    .unwrap()
17604                    .connection_id,
17605                ConnectionId::new(4)
17606            );
17607            assert_eq!(
17608                registry.get_module("aft").unwrap().unwrap().connection_id,
17609                ConnectionId::new(1),
17610                "the candidate must not take the active slot"
17611            );
17612            let replayed = handler
17613                .handle_control(
17614                    ConnectionId::new(5),
17615                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17616                )
17617                .unwrap();
17618            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17619
17620            // The case only this gate covers: the incumbent has died mid-swap,
17621            // so its duplicate refusal is gone too, and without the gate a
17622            // key-holder would take the id's ACTIVE slot.
17623            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17624            let squatter = handler
17625                .handle_control(
17626                    ConnectionId::new(6),
17627                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17628                )
17629                .unwrap();
17630            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17631            assert!(
17632                registry.get_module("aft").unwrap().is_none(),
17633                "a squatter took the active slot of an id being swapped"
17634            );
17635        }
17636
17637        /// Design mutation arm (iii). A reserved module's candidate presents a
17638        /// nonce the reserved gate has never seen (that gate holds the
17639        /// incumbent's), so the swap gate must run first or the candidate is
17640        /// refused `reserved_module` and a reserved module can never be swapped.
17641        #[test]
17642        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17643            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17644
17645            let replies = handler
17646                .handle_control(
17647                    ConnectionId::new(2),
17648                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17649                )
17650                .unwrap();
17651
17652            assert_eq!(
17653                replies[0].header.ty,
17654                FrameType::HelloAck,
17655                "reserved candidate refused: {:?}",
17656                serde_json::from_slice::<Value>(&replies[0].body).ok()
17657            );
17658            assert_eq!(
17659                registry
17660                    .get_candidate("vault")
17661                    .unwrap()
17662                    .unwrap()
17663                    .connection_id,
17664                ConnectionId::new(2)
17665            );
17666        }
17667
17668        /// With no swap open the gate is inert: the incumbent's reserved gate
17669        /// and duplicate refusal behave exactly as before.
17670        #[test]
17671        fn without_an_open_swap_the_ordinary_gates_decide() {
17672            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17673            supervisor.close_swap("vault");
17674
17675            let candidate = handler
17676                .handle_control(
17677                    ConnectionId::new(2),
17678                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17679                )
17680                .unwrap();
17681            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17682            let duplicate = handler
17683                .handle_control(
17684                    ConnectionId::new(3),
17685                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17686                )
17687                .unwrap();
17688            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17689            assert!(registry.get_candidate("vault").unwrap().is_none());
17690        }
17691    }
17692
17693    /// `scope.sync` and `scope.describe` through the real control handler: who
17694    /// may sync is decided by the registration and launch nonce of the module
17695    /// connection, never by the request body.
17696    mod scopes {
17697        mod idle_tests {
17698            use super::*;
17699            include!("scope_idle_tests.rs");
17700        }
17701        mod route_open_timings {
17702            use super::*;
17703            include!("route_open_timing_tests.rs");
17704        }
17705        mod route_open_refusal_log {
17706            use super::*;
17707            include!("route_open_refusal_log_tests.rs");
17708        }
17709        mod activity {
17710            include!("activity_scope_tests.rs");
17711        }
17712        mod delta {
17713            include!("scope_handler_delta_tests.rs");
17714        }
17715        mod call_modes {
17716            include!("scope_call_mode_tests.rs");
17717        }
17718        mod federated {
17719            include!("scope_federated_tests.rs");
17720        }
17721        #[tokio::test]
17722        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17723            let rig = rig().await;
17724            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17725                "s".into(),
17726                1,
17727                "tool".into(),
17728            ))
17729            .unwrap();
17730            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17731            let (direct, _rx) = wide_ctx(999);
17732            for (ctx, expected) in [
17733                (&rig.owner, FrameType::Response),
17734                (&direct, FrameType::Error),
17735            ] {
17736                let frame =
17737                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17738                        .unwrap();
17739                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17740                assert_eq!(replies[0].header.ty, expected);
17741                if expected == FrameType::Response {
17742                    let reply: subc_control::ScopeServedByReply =
17743                        serde_json::from_slice(&replies[0].body).unwrap();
17744                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17745                } else {
17746                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17747                }
17748            }
17749            let reply = call(
17750                &rig.handler,
17751                &rig.owner,
17752                &ModuleControlRequestFromModule::ScopeDescribe {
17753                    owner: Principal::Reserved {
17754                        module_id: OWNER.into(),
17755                    },
17756                    scope_ref: "s".into(),
17757                },
17758            )
17759            .await;
17760            assert!(matches!(
17761                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17762                ModuleControlResponseToModule::ScopeDescribe { .. }
17763            ));
17764        }
17765
17766        use subc_protocol::scope::{
17767            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17768            ScopeStatus,
17769        };
17770
17771        use super::*;
17772
17773        const OWNER: &str = "prefrontal-core";
17774
17775        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17776            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17777        }
17778
17779        async fn call(
17780            handler: &ControlHandler,
17781            ctx: &RouteCtx,
17782            request: &ModuleControlRequestFromModule,
17783        ) -> Frame {
17784            let body = serde_json::to_vec(request).unwrap();
17785            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17786            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17787            assert_eq!(replies.len(), 1, "{replies:?}");
17788            replies.pop().unwrap()
17789        }
17790
17791        async fn sync(
17792            handler: &ControlHandler,
17793            ctx: &RouteCtx,
17794            generation: u64,
17795            scopes: Vec<ScopeRecord>,
17796        ) -> Result<ModuleControlResponseToModule, String> {
17797            let reply = call(
17798                handler,
17799                ctx,
17800                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17801            )
17802            .await;
17803            match reply.header.ty {
17804                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17805                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17806            }
17807        }
17808
17809        async fn describe(
17810            handler: &ControlHandler,
17811            ctx: &RouteCtx,
17812            owner: &str,
17813            scope_ref: &str,
17814        ) -> ModuleControlResponseToModule {
17815            let reply = call(
17816                handler,
17817                ctx,
17818                &ModuleControlRequestFromModule::ScopeDescribe {
17819                    owner: Principal::Reserved {
17820                        module_id: owner.to_string(),
17821                    },
17822                    scope_ref: scope_ref.to_string(),
17823                },
17824            )
17825            .await;
17826            assert_eq!(
17827                reply.header.ty,
17828                FrameType::Response,
17829                "{:?}",
17830                parse_error(&reply)
17831            );
17832            serde_json::from_slice(&reply.body).unwrap()
17833        }
17834
17835        #[tokio::test]
17836        async fn scope_describe_never_echoes_bind_proof_or_hash() {
17837            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17838            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17839            let mut rig = rig().await;
17840            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17841                .await;
17842            let _bound = rig
17843                .bound(
17844                    Some(AFT),
17845                    PLEXUS,
17846                    Some(
17847                        rig.selector("s", Some(1))
17848                            .with_bind_proof(Some(PROOF.into())),
17849                    ),
17850                )
17851                .await;
17852            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17853            let output = serde_json::to_string(&reply).unwrap();
17854            assert!(output.contains("\"status\":\"live\""));
17855            assert!(!output.contains(PROOF));
17856            assert!(!output.contains(HASH));
17857        }
17858
17859        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
17860        async fn module(
17861            handler: &ControlHandler,
17862            connection: u64,
17863            module_id: &str,
17864            nonce: Option<&str>,
17865        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17866            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17867            hello_via_sink(
17868                handler,
17869                &ctx,
17870                &mut rx,
17871                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17872            )
17873            .await;
17874            (ctx, rx)
17875        }
17876
17877        /// `direct` and every other client connection has no registration, so
17878        /// it can neither sync nor own a scope.
17879        #[tokio::test]
17880        async fn a_client_connection_cannot_sync_or_describe() {
17881            let handler = ControlHandler::new(Arc::new(Registry::default()));
17882            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17883            for request in [
17884                ModuleControlRequestFromModule::ScopeSync {
17885                    generation: 1,
17886                    scopes: vec![head("s", 1)],
17887                },
17888                ModuleControlRequestFromModule::ScopeDescribe {
17889                    owner: Principal::Direct,
17890                    scope_ref: "s".to_string(),
17891                },
17892            ] {
17893                let reply = call(&handler, &ctx, &request).await;
17894                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17895            }
17896            assert!(
17897                !handler
17898                    .scopes
17899                    .read()
17900                    .unwrap()
17901                    .describe(
17902                        &Principal::Reserved {
17903                            module_id: OWNER.to_string()
17904                        },
17905                        "s"
17906                    )
17907                    .owner_synced
17908            );
17909        }
17910
17911        /// A module the supervisor did not spawn registers without a launch
17912        /// nonce, so it is never an owner's current launch.
17913        #[tokio::test]
17914        async fn a_module_without_a_supervised_launch_cannot_sync() {
17915            let handler = ControlHandler::new(Arc::new(Registry::default()));
17916            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17917            assert_eq!(
17918                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17919                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17920            );
17921        }
17922
17923        #[tokio::test]
17924        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17925            let supervisor = SupervisorHandle::new();
17926            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17927            let handler = ControlHandler::new(Arc::new(Registry::default()))
17928                .with_supervisor(supervisor.clone());
17929            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17930            sync(&handler, &incumbent, 1, vec![head("s", 1)])
17931                .await
17932                .expect("the current launch syncs");
17933
17934            // A swap candidate registers with the swap token and is refused
17935            // while the incumbent keeps syncing.
17936            supervisor.open_swap(OWNER, "n2".to_string());
17937            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17938            assert_eq!(
17939                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17940                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17941            );
17942            sync(&handler, &incumbent, 2, vec![head("s", 1)])
17943                .await
17944                .expect("the serving owner syncs during the swap");
17945
17946            // The swap fails and is rolled back. The candidate never held sync
17947            // authority, and still cannot sync.
17948            supervisor.close_swap(OWNER);
17949            assert_eq!(
17950                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17951                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17952            );
17953            sync(&handler, &incumbent, 3, vec![head("s", 1)])
17954                .await
17955                .expect("the serving owner syncs after the rollback");
17956            handler.cleanup_connection(candidate.connection_id).unwrap();
17957
17958            // A swap that cuts over. Promotion records the candidate's nonce as
17959            // the module's spawn nonce, which is what `set_spawn_nonce` does
17960            // here; the promoted connection then takes authority at any
17961            // generation and the superseded incumbent is refused.
17962            supervisor.open_swap(OWNER, "n3".to_string());
17963            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17964            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17965            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17966                .await
17967                .expect("the promoted launch takes authority");
17968            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17969                panic!("unexpected reply {reply:?}");
17970            };
17971            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17972            assert_eq!(
17973                sync(&handler, &incumbent, 4, Vec::new()).await,
17974                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17975            );
17976        }
17977
17978        /// Authority dies with its connection: the cleanup path releases it,
17979        /// so the owner's next connection takes it at any generation.
17980        #[tokio::test]
17981        async fn closing_the_authority_connection_frees_sync_authority() {
17982            let supervisor = SupervisorHandle::new();
17983            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17984            let handler = ControlHandler::new(Arc::new(Registry::default()))
17985                .with_supervisor(supervisor.clone());
17986            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17987            sync(&handler, &first, 10, vec![head("s", 1)])
17988                .await
17989                .unwrap();
17990            handler.cleanup_connection(first.connection_id).unwrap();
17991
17992            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17993            sync(&handler, &second, 1, vec![head("s", 1)])
17994                .await
17995                .expect("the next connection takes the released authority");
17996        }
17997
17998        #[tokio::test]
17999        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18000            let supervisor = SupervisorHandle::new();
18001            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18002            let handler =
18003                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18004            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18005            sync(&handler, &first, 10, vec![head("s", 1)])
18006                .await
18007                .unwrap();
18008            handler
18009                .handle_control_frame(
18010                    &first,
18011                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18012                )
18013                .await
18014                .unwrap();
18015            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18016            sync(&handler, &second, 1, vec![head("s", 1)])
18017                .await
18018                .expect("GOODBYE releases authority even if the old socket remains open");
18019        }
18020
18021        #[tokio::test]
18022        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18023            let registry = Arc::new(Registry::default());
18024            let supervisor_handle = SupervisorHandle::new();
18025            let supervisor =
18026                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
18027                    .with_handle(supervisor_handle.clone())
18028                    .with_daemon_incarnation("incarnation-7".to_string());
18029            // Configured with enabled: false, so the supervisor lists the
18030            // module without spawning a process for it.
18031            supervisor
18032                .supervise_configured(
18033                    ModuleSpec {
18034                        carry_containment: false,
18035                        module_id: OWNER.to_string(),
18036                        program: PathBuf::from("/nonexistent/prefrontal-core"),
18037                        args: Vec::new(),
18038                        env: Vec::new(),
18039                        reserved: false,
18040                        reserved_prefixes: Vec::new(),
18041                        protocol: ModuleProtocol::Subc,
18042                        overlap: Default::default(),
18043                    },
18044                    false,
18045                )
18046                .unwrap();
18047            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18048            let handler =
18049                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
18050            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18051
18052            // Configured but not yet synced: a reader waits for the owner.
18053            let ModuleControlResponseToModule::ScopeDescribe {
18054                status,
18055                daemon_incarnation,
18056                owner_synced,
18057                owner_configured,
18058                scope,
18059                ..
18060            } = describe(&handler, &reader, OWNER, "s").await
18061            else {
18062                panic!("not a describe reply");
18063            };
18064            assert_eq!(status, ScopeStatus::NotLive);
18065            assert_eq!(daemon_incarnation, "incarnation-7");
18066            assert!(!owner_synced);
18067            assert!(owner_configured);
18068            assert!(scope.is_none());
18069
18070            // Not a supervised module: the owner will never sync, and a reader
18071            // refuses rather than waits.
18072            let ModuleControlResponseToModule::ScopeDescribe {
18073                status,
18074                owner_configured,
18075                ..
18076            } = describe(&handler, &reader, "ghost", "s").await
18077            else {
18078                panic!("not a describe reply");
18079            };
18080            assert_eq!(status, ScopeStatus::NotLive);
18081            assert!(!owner_configured);
18082
18083            // Live, with the stamp fields and the computed owner_authorized.
18084            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18085            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18086            let ModuleControlResponseToModule::ScopeDescribe {
18087                status,
18088                scope_epoch,
18089                owner_synced,
18090                scope,
18091                ..
18092            } = describe(&handler, &reader, OWNER, "s").await
18093            else {
18094                panic!("not a describe reply");
18095            };
18096            assert_eq!(status, ScopeStatus::Live);
18097            assert_eq!(scope_epoch, Some(4));
18098            assert!(owner_synced);
18099            let stamp = scope.expect("a live scope carries its stamp");
18100            assert!(
18101                stamp.owner_authorized,
18102                "prefrontal-core is the default authority"
18103            );
18104            assert_eq!(stamp.kind, ScopeKind::Head);
18105        }
18106
18107        #[tokio::test]
18108        async fn scope_authority_owners_decides_owner_authorized() {
18109            let supervisor = SupervisorHandle::new();
18110            supervisor.set_spawn_nonce("broca", "b1".to_string());
18111            let handler = ControlHandler::new(Arc::new(Registry::default()))
18112                .with_supervisor(supervisor)
18113                .with_scope_authority_owners(vec!["broca".to_string()]);
18114            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18115            let mut gated = head("s", 1);
18116            gated.attributes.agent_id = Some("agent".to_string());
18117            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18118            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18119                describe(&handler, &broca, "broca", "s").await
18120            else {
18121                panic!("not a describe reply");
18122            };
18123            assert!(scope.unwrap().owner_authorized);
18124        }
18125
18126        /// With route admission, the stamp, the commit re-check and drains in
18127        /// place, the feature is advertised: the module ops in HELLO_ACK, and
18128        /// `scopes/v1` in HELLO_ACK and `server.describe`.
18129        #[tokio::test]
18130        async fn scope_ops_and_the_scopes_capability_are_advertised() {
18131            let handler = ControlHandler::new(Arc::new(Registry::default()));
18132            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18133            let ack = hello_via_sink(
18134                &handler,
18135                &ctx,
18136                &mut rx,
18137                hello_frame("m", PROTOCOL_VERSION, 1),
18138            )
18139            .await;
18140            let ack = parse_ack(&ack);
18141            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18142                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18143            }
18144            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18145
18146            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18147            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18148            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18149            let reply = handler
18150                .handle_control_frame(&client, frame)
18151                .await
18152                .unwrap()
18153                .pop()
18154                .unwrap();
18155            let ClientControlResponse::ServerDescribe { capabilities, .. } =
18156                serde_json::from_slice(&reply.body).unwrap()
18157            else {
18158                panic!("not a server.describe reply");
18159            };
18160            assert!(
18161                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18162                "{capabilities:?}"
18163            );
18164        }
18165
18166        // ---- route admission, stamps, commit re-check and drains ----------
18167
18168        const PLEXUS: &str = "plexus";
18169        const OTHER: &str = "other";
18170        const AFT: &str = "aft";
18171        const BROCA: &str = "broca";
18172        const MAGIC: &str = "magic-context";
18173
18174        fn nonce(module_id: &str) -> String {
18175            format!("nonce-{module_id}")
18176        }
18177
18178        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18179            let (tx, rx) = mpsc::channel(64);
18180            (
18181                RouteCtx {
18182                    connection_id: ConnectionId::new(connection),
18183                    egress: FrameSink::new(tx),
18184                },
18185                rx,
18186            )
18187        }
18188
18189        /// A daemon with a configured owner (prefrontal-core) registered on its
18190        /// own module connection, two routable targets (plexus, other), and
18191        /// launch nonces minted for the modules that open routes as carriers.
18192        struct Rig {
18193            handler: ControlHandler,
18194            forwarding: Arc<ForwardingTable>,
18195            owner: RouteCtx,
18196            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18197            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18198            generation: u64,
18199            next_connection: u64,
18200            _supervisor: Supervisor,
18201        }
18202
18203        async fn rig() -> Rig {
18204            rig_with_flow_support(true).await
18205        }
18206
18207        async fn rig_with_flow_support(flow_support: bool) -> Rig {
18208            rig_with_scope_capabilities(flow_support, false).await
18209        }
18210
18211        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18212            let mut provides = Vec::new();
18213            if flow_support {
18214                provides.push("flow-scopes/v1");
18215            }
18216            if run_support {
18217                provides.push("agent-run-scopes/v1");
18218            }
18219            rig_with_target_capabilities(&provides).await
18220        }
18221
18222        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18223            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18224            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18225            // Decoder versions alone cannot admit scoped calls: only a capability
18226            // declaration promises the required run or flow behaviour.
18227            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18228            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18229            Frame::build(
18230                FrameType::Hello,
18231                control_flags(),
18232                0,
18233                0,
18234                connection,
18235                serde_json::to_vec(&body).unwrap(),
18236            )
18237            .unwrap()
18238        }
18239
18240        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18241            let registry = Arc::new(Registry::default());
18242            let forwarding = Arc::new(ForwardingTable::default());
18243            let supervisor_handle = SupervisorHandle::new();
18244            let supervisor =
18245                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
18246                    .with_handle(supervisor_handle.clone());
18247            supervisor
18248                .supervise_configured(
18249                    ModuleSpec {
18250                        carry_containment: false,
18251                        module_id: OWNER.to_string(),
18252                        program: PathBuf::from("/nonexistent/prefrontal-core"),
18253                        args: Vec::new(),
18254                        env: Vec::new(),
18255                        reserved: false,
18256                        reserved_prefixes: Vec::new(),
18257                        protocol: ModuleProtocol::Subc,
18258                        overlap: Default::default(),
18259                    },
18260                    false,
18261                )
18262                .unwrap();
18263            for module_id in [OWNER, AFT, BROCA, MAGIC] {
18264                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18265            }
18266            let handler =
18267                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18268                    .with_supervisor(supervisor_handle);
18269            let (owner, mut owner_rx) = wide_ctx(1);
18270            hello_via_sink(
18271                &handler,
18272                &owner,
18273                &mut owner_rx,
18274                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18275            )
18276            .await;
18277            let mut modules = BTreeMap::new();
18278            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18279                let (ctx, mut rx) = wide_ctx(connection);
18280                let hello = hello_with_capabilities(module_id, connection, provides);
18281                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18282                modules.insert(module_id.to_string(), (ctx, rx));
18283            }
18284            Rig {
18285                handler,
18286                forwarding,
18287                owner,
18288                _owner_rx: owner_rx,
18289                modules,
18290                generation: 0,
18291                next_connection: 100,
18292                _supervisor: supervisor,
18293            }
18294        }
18295
18296        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18297            ScopeCarrier::new(Principal::Reserved {
18298                module_id: module_id.to_string(),
18299            })
18300            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18301        }
18302
18303        /// The scope most tests open under: aft carries to any module, broca
18304        /// only to plexus and other, and the owner delegates as agent-1.
18305        fn session(scope_epoch: u64) -> ScopeRecord {
18306            let mut record = head("s", scope_epoch);
18307            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18308            record.attributes.agent_id = Some("agent-1".to_string());
18309            record.attributes.delegates = true;
18310            record
18311        }
18312
18313        impl Rig {
18314            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18315                self.generation += 1;
18316                sync(&self.handler, &self.owner, self.generation, scopes)
18317                    .await
18318                    .expect("the owner's sync is accepted");
18319            }
18320
18321            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18322                ScopeSelector::new(
18323                    Principal::Reserved {
18324                        module_id: OWNER.to_string(),
18325                    },
18326                    scope_ref.to_string(),
18327                    scope_epoch,
18328                )
18329            }
18330
18331            fn open_frame(
18332                &mut self,
18333                opener: Option<&str>,
18334                target: &str,
18335                scope: Option<ScopeSelector>,
18336            ) -> (
18337                RouteCtx,
18338                mpsc::Receiver<crate::router::OutboundFrame>,
18339                Frame,
18340            ) {
18341                self.next_connection += 1;
18342                let (ctx, rx) = wide_ctx(self.next_connection);
18343                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18344                    busy_refusals: None,
18345                    target: RouteTarget::ToolProvider {
18346                        module_id: target.to_string(),
18347                    },
18348                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18349                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
18350                        module_id: module_id.to_string(),
18351                        launch_nonce: nonce(module_id),
18352                    }),
18353                    consumer_capabilities: None,
18354                    role_versions: None,
18355                    admission_facts: None,
18356                    scope,
18357                })
18358                .unwrap();
18359                let frame = Frame::build(
18360                    FrameType::Request,
18361                    control_flags(),
18362                    0,
18363                    0,
18364                    self.next_connection,
18365                    body,
18366                )
18367                .unwrap();
18368                (ctx, rx, frame)
18369            }
18370
18371            /// Open and expect a refusal before anything is relayed.
18372            async fn refused(
18373                &mut self,
18374                opener: Option<&str>,
18375                target: &str,
18376                scope: Option<ScopeSelector>,
18377            ) -> String {
18378                self.refusal_body(opener, target, scope).await["code"]
18379                    .as_str()
18380                    .unwrap()
18381                    .to_string()
18382            }
18383
18384            async fn refusal_body(
18385                &mut self,
18386                opener: Option<&str>,
18387                target: &str,
18388                scope: Option<ScopeSelector>,
18389            ) -> Value {
18390                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18391                let replies = tokio::time::timeout(
18392                    Duration::from_secs(2),
18393                    self.handler.handle_control_frame(&ctx, frame),
18394                )
18395                .await
18396                .expect("the open must be refused before waiting for a bind ack")
18397                .unwrap();
18398                assert_eq!(replies.len(), 1, "{replies:?}");
18399                assert_eq!(replies[0].header.ty, FrameType::Error);
18400                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18401                assert!(
18402                    module_rx.try_recv().is_err(),
18403                    "a refused open relays nothing"
18404                );
18405                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18406                parse_error(&replies[0])
18407            }
18408
18409            /// Start an open and return its task and the bind the target got.
18410            async fn relayed(
18411                &mut self,
18412                opener: Option<&str>,
18413                target: &str,
18414                scope: Option<ScopeSelector>,
18415            ) -> Relayed {
18416                self.relayed_with_legacy_root(opener, target, scope, None)
18417                    .await
18418            }
18419
18420            async fn relayed_with_legacy_root(
18421                &mut self,
18422                opener: Option<&str>,
18423                target: &str,
18424                scope: Option<ScopeSelector>,
18425                legacy_root: Option<&std::path::Path>,
18426            ) -> Relayed {
18427                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18428                if let Some(root) = legacy_root {
18429                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18430                    body["identity"]["project_root"] = serde_json::json!(root);
18431                    frame.body = serde_json::to_vec(&body).unwrap();
18432                }
18433                let handler = self.handler.clone();
18434                let task_ctx = ctx.clone();
18435                let task = tokio::spawn(async move {
18436                    handler
18437                        .handle_control_frame(&task_ctx, frame)
18438                        .await
18439                        .unwrap()
18440                });
18441                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18442                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18443                    .await
18444                    .expect("the target receives the relayed route.bind")
18445                    .unwrap()
18446                    .frame;
18447                Relayed {
18448                    target: target.to_string(),
18449                    client: ctx,
18450                    client_rx: rx,
18451                    task,
18452                    bind,
18453                }
18454            }
18455
18456            async fn ack(&self, relayed: &Relayed) {
18457                let (module, _) = &self.modules[&relayed.target];
18458                self.handler
18459                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18460                    .await
18461                    .unwrap();
18462            }
18463
18464            /// Open, ack and return the bound route.
18465            async fn bound(
18466                &mut self,
18467                opener: Option<&str>,
18468                target: &str,
18469                scope: Option<ScopeSelector>,
18470            ) -> Bound {
18471                let relayed = self.relayed(opener, target, scope).await;
18472                self.commit_relayed(relayed).await
18473            }
18474
18475            async fn bound_with_legacy_root(
18476                &mut self,
18477                opener: Option<&str>,
18478                target: &str,
18479                scope: Option<ScopeSelector>,
18480                root: &std::path::Path,
18481            ) -> Bound {
18482                let relayed = self
18483                    .relayed_with_legacy_root(opener, target, scope, Some(root))
18484                    .await;
18485                self.commit_relayed(relayed).await
18486            }
18487
18488            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18489                self.ack(&relayed).await;
18490                let Relayed {
18491                    target,
18492                    client,
18493                    mut client_rx,
18494                    task,
18495                    bind,
18496                } = relayed;
18497                assert!(
18498                    task.await.unwrap().is_empty(),
18499                    "the open is answered by commit"
18500                );
18501                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18502                Bound {
18503                    target,
18504                    client,
18505                    client_rx,
18506                    channel,
18507                    epoch,
18508                    bind,
18509                }
18510            }
18511
18512            fn live(&self, route: &Bound) -> bool {
18513                matches!(
18514                    self.forwarding
18515                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18516                        .unwrap(),
18517                    DataRoute::Client(DataRouteState::Bound(_))
18518                )
18519            }
18520        }
18521
18522        struct Relayed {
18523            target: String,
18524            client: RouteCtx,
18525            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18526            task: tokio::task::JoinHandle<Vec<Frame>>,
18527            bind: Frame,
18528        }
18529
18530        struct Bound {
18531            target: String,
18532            client: RouteCtx,
18533            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18534            channel: u16,
18535            epoch: u32,
18536            bind: Frame,
18537        }
18538
18539        impl Bound {
18540            /// The reason of the `route.closed` this client was sent, after
18541            /// checking it also got a GOODBYE on exactly this route.
18542            fn closed_reason(&mut self) -> RouteCloseReason {
18543                let mut reason = None;
18544                let mut goodbye = false;
18545                while let Ok(outbound) = self.client_rx.try_recv() {
18546                    let frame = outbound.frame;
18547                    match frame.header.ty {
18548                        FrameType::Goodbye => {
18549                            assert_eq!(
18550                                (frame.header.channel, frame.header.epoch),
18551                                (self.channel, self.epoch)
18552                            );
18553                            goodbye = true;
18554                        }
18555                        FrameType::Push => {
18556                            let ClientControlPush::RouteClosed {
18557                                reason: r,
18558                                module_id,
18559                                ..
18560                            } = serde_json::from_slice(&frame.body).unwrap()
18561                            else {
18562                                panic!("unexpected push");
18563                            };
18564                            assert_eq!(module_id, self.target);
18565                            reason = Some(r);
18566                        }
18567                        other => panic!("unexpected frame {other:?}"),
18568                    }
18569                }
18570                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18571                reason.expect("the client is told why the route closed")
18572            }
18573
18574            fn untouched(&mut self) -> bool {
18575                self.client_rx.try_recv().is_err()
18576            }
18577
18578            fn stamp(&self) -> Option<ScopeStamp> {
18579                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18580                    ModuleControlRequest::RouteBind { scope, .. } => scope,
18581                    other => panic!("expected a route.bind, got {other:?}"),
18582                }
18583            }
18584        }
18585
18586        #[tokio::test]
18587        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18588        ) {
18589            let mut rig = rig().await;
18590            let mut record = session(1);
18591            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18592            record.child_owners = vec![Principal::Reserved {
18593                module_id: MAGIC.to_string(),
18594            }];
18595            rig.sync(vec![record]).await;
18596            let scope = || Some(rig_selector("s", Some(1)));
18597
18598            // Admitted: the owner, a bare carrier to any module, a targeted
18599            // carrier to its listed module.
18600            rig.bound(Some(OWNER), PLEXUS, scope()).await;
18601            rig.bound(Some(AFT), OTHER, scope()).await;
18602            rig.bound(Some(BROCA), PLEXUS, scope()).await;
18603
18604            // Refused scope_not_carrier: a targeted carrier to an unlisted
18605            // module, a module that is not listed at all (a child owner is not
18606            // a carrier), and a direct key-holder.
18607            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18608                assert_eq!(
18609                    rig.refused(opener, target, scope()).await,
18610                    error_codes::SCOPE_NOT_CARRIER,
18611                    "{opener:?} -> {target}"
18612                );
18613            }
18614        }
18615
18616        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18617            ScopeSelector::new(
18618                Principal::Reserved {
18619                    module_id: OWNER.to_string(),
18620                },
18621                scope_ref.to_string(),
18622                scope_epoch,
18623            )
18624        }
18625
18626        #[tokio::test]
18627        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18628            let mut rig = rig().await;
18629            rig.sync(vec![session(1)]).await;
18630            for opener in [OWNER, AFT] {
18631                assert_eq!(
18632                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18633                        .await,
18634                    error_codes::SCOPE_EPOCH_REQUIRED,
18635                    "{opener}"
18636                );
18637            }
18638        }
18639
18640        #[tokio::test]
18641        async fn admission_separates_not_synced_not_live_and_ended() {
18642            let mut rig = rig().await;
18643            // Before the configured owner's first sync: retryable.
18644            let code = rig
18645                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18646                .await;
18647            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18648            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18649
18650            // An owner that is not configured will never sync: terminal.
18651            let ghost = ScopeSelector::new(
18652                Principal::Reserved {
18653                    module_id: "ghost".to_string(),
18654                },
18655                "s".to_string(),
18656                Some(1),
18657            );
18658            assert_eq!(
18659                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18660                error_codes::SCOPE_NOT_LIVE
18661            );
18662
18663            rig.sync(vec![session(2)]).await;
18664            assert_eq!(
18665                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18666                    .await,
18667                error_codes::SCOPE_NOT_LIVE
18668            );
18669            for epoch in [1, 3] {
18670                assert_eq!(
18671                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18672                        .await,
18673                    error_codes::SCOPE_ENDED,
18674                    "epoch {epoch}"
18675                );
18676            }
18677            // Control: the live epoch is admitted.
18678            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18679                .await;
18680        }
18681
18682        #[tokio::test]
18683        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18684            let mut rig = rig().await;
18685            rig.sync(vec![session(1)]).await;
18686            let route = rig
18687                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18688                .await;
18689            let stamp = route.stamp().expect("a scoped bind carries the stamp");
18690            assert_eq!(stamp.scope_ref, "s");
18691            assert_eq!(stamp.scope_epoch, 1);
18692            assert_eq!(stamp.kind, ScopeKind::Head);
18693            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18694            assert!(stamp.attributes.delegates);
18695            assert!(stamp.owner_authorized);
18696            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18697            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18698
18699            // broca owns a scope of its own on its own module connection; it is
18700            // not in scope_authority_owners, so its stamp is not authorized.
18701            let (broca, mut broca_rx) = wide_ctx(50);
18702            hello_via_sink(
18703                &rig.handler,
18704                &broca,
18705                &mut broca_rx,
18706                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18707            )
18708            .await;
18709            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18710                .await
18711                .unwrap();
18712            let own = ScopeSelector::new(
18713                Principal::Reserved {
18714                    module_id: BROCA.to_string(),
18715                },
18716                "b".to_string(),
18717                Some(1),
18718            );
18719            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18720            assert!(!route.stamp().unwrap().owner_authorized);
18721        }
18722
18723        #[tokio::test]
18724        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18725            let mut rig = rig().await;
18726            let mut record = head("s", 1);
18727            record.carriers = vec![carrier(AFT, None)];
18728            let flow_id = "Flow:run-7/step_2!~";
18729            record.attributes.flow_id = Some(flow_id.to_string());
18730            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18731                .await
18732                .unwrap();
18733            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18734                panic!("not a sync reply");
18735            };
18736            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18737            let route = rig
18738                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18739                .await;
18740            assert!(rig.live(&route), "the stamped bind committed");
18741            let stamp = route.stamp().expect("a flow scope carries a stamp");
18742            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18743            assert_eq!(stamp.attributes.agent_id, None);
18744            assert!(!stamp.attributes.delegates);
18745            assert!(stamp.owner_authorized);
18746            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18747            assert_eq!(unscoped.stamp(), None);
18748        }
18749
18750        #[tokio::test]
18751        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18752            let mut rig = rig_with_flow_support(false).await;
18753            let mut record = session(1);
18754            record.attributes.flow_id = Some("flow:7".to_string());
18755            rig.sync(vec![record]).await;
18756            for opener in [OWNER, AFT] {
18757                let body = rig
18758                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18759                    .await;
18760                assert_eq!(body["code"], "target_flow_unsupported");
18761                let message = body["message"].as_str().unwrap();
18762                for required in [PLEXUS, "flow-scopes/v1"] {
18763                    assert!(message.contains(required), "{message}");
18764                }
18765            }
18766        }
18767
18768        #[tokio::test]
18769        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18770            let mut rig = rig_with_flow_support(true).await;
18771            let mut record = session(1);
18772            record.attributes.flow_id = Some("flow:7".to_string());
18773            rig.sync(vec![record]).await;
18774            for opener in [OWNER, AFT] {
18775                let route = rig
18776                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18777                    .await;
18778                assert!(rig.live(&route));
18779                assert_eq!(
18780                    route.stamp().unwrap().attributes.flow_id.as_deref(),
18781                    Some("flow:7")
18782                );
18783            }
18784        }
18785
18786        #[tokio::test]
18787        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18788            use std::future::Future;
18789
18790            let mut rig = rig_with_flow_support(true).await;
18791            let mut record = session(1);
18792            record.attributes.flow_id = Some("flow:7".to_string());
18793            rig.sync(vec![record]).await;
18794            let (client, mut client_rx, frame) =
18795                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18796            // Hold the route-open response permit so admission sees the first
18797            // target but relay reservation cannot capture an endpoint yet.
18798            for _ in 0..64 {
18799                client.egress.try_send(route_bind_ack(1)).unwrap();
18800            }
18801            let handler = rig.handler.clone();
18802            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18803            std::future::poll_fn(|cx| {
18804                assert!(open.as_mut().poll(cx).is_pending());
18805                std::task::Poll::Ready(())
18806            })
18807            .await;
18808            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18809
18810            let old_connection = rig.modules[PLEXUS].0.connection_id;
18811            rig.handler.cleanup_connection(old_connection).unwrap();
18812            let (replacement, mut replacement_rx) = wide_ctx(200);
18813            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18814            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18815            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18816            let hello = Frame::build(
18817                FrameType::Hello,
18818                control_flags(),
18819                0,
18820                0,
18821                200,
18822                serde_json::to_vec(&body).unwrap(),
18823            )
18824            .unwrap();
18825            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18826
18827            client_rx.try_recv().unwrap();
18828            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18829                .await
18830                .expect("the replacement is refused without waiting for a bind ack")
18831                .unwrap();
18832            assert_eq!(replies.len(), 1);
18833            let body = parse_error(&replies[0]);
18834            assert_eq!(body["code"], "target_flow_unsupported");
18835            for required in [PLEXUS, "flow-scopes/v1"] {
18836                assert!(body["message"].as_str().unwrap().contains(required));
18837            }
18838            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18839            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18840            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18841            assert!(rig
18842                .handler
18843                .registry
18844                .get_module_by_connection(replacement.connection_id)
18845                .unwrap()
18846                .is_some());
18847        }
18848
18849        #[tokio::test]
18850        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18851            use std::future::Future;
18852            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18853            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18854            const NEXT_HASH: &str =
18855                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18856            let mut rig = rig().await;
18857            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18858            record.carriers.push(ScopeCarrier::new(Principal::Direct));
18859            rig.sync(vec![record.clone()]).await;
18860            let (client, mut rx, frame) = rig.open_frame(
18861                None,
18862                PLEXUS,
18863                Some(
18864                    rig.selector("s", Some(1))
18865                        .with_bind_proof(Some(PROOF.into())),
18866                ),
18867            );
18868            // Hold egress reservation while the scope owner changes its bind-proof
18869            // hash, so a proof valid at admission must be checked again at relay.
18870            for _ in 0..64 {
18871                client.egress.try_send(route_bind_ack(1)).unwrap();
18872            }
18873            let handler = rig.handler.clone();
18874            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18875            std::future::poll_fn(|cx| {
18876                assert!(open.as_mut().poll(cx).is_pending());
18877                std::task::Poll::Ready(())
18878            })
18879            .await;
18880            record.bind_proof_sha256 = Some(NEXT_HASH.into());
18881            rig.sync(vec![record]).await;
18882            rx.try_recv().unwrap();
18883            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18884                .await
18885                .unwrap()
18886                .unwrap();
18887            assert_eq!(
18888                parse_error(&replies[0])["code"],
18889                error_codes::SCOPE_BIND_PROOF_MISMATCH
18890            );
18891            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18892            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18893            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18894        }
18895
18896        #[tokio::test]
18897        async fn scope_bind_proof_reserved_openers_ignore_proof() {
18898            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18899            let mut rig = rig().await;
18900            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18901                .await;
18902            for opener in [OWNER, AFT] {
18903                for proof in [None, Some("malformed+proof=")] {
18904                    let route = rig
18905                        .bound(
18906                            Some(opener),
18907                            PLEXUS,
18908                            Some(
18909                                rig.selector("s", Some(1))
18910                                    .with_bind_proof(proof.map(str::to_string)),
18911                            ),
18912                        )
18913                        .await;
18914                    assert!(rig.live(&route));
18915                }
18916            }
18917        }
18918
18919        #[tokio::test]
18920        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18921            use std::future::Future;
18922            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18923            const NEXT_HASH: &str =
18924                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18925            let mut rig = rig().await;
18926            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18927                .await;
18928            let (client, mut rx, frame) =
18929                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18930            for _ in 0..64 {
18931                client.egress.try_send(route_bind_ack(1)).unwrap();
18932            }
18933            let handler = rig.handler.clone();
18934            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18935            std::future::poll_fn(|cx| {
18936                assert!(open.as_mut().poll(cx).is_pending());
18937                std::task::Poll::Ready(())
18938            })
18939            .await;
18940            rig.sync(vec![
18941                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18942            ])
18943            .await;
18944            rx.try_recv().unwrap();
18945            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18946            let bind = std::future::poll_fn(|cx| {
18947                assert!(open.as_mut().poll(cx).is_pending());
18948                module_rx.poll_recv(cx)
18949            })
18950            .await
18951            .unwrap()
18952            .frame;
18953            handler
18954                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18955                .await
18956                .unwrap();
18957            let replies = tokio::time::timeout(Duration::from_secs(2), open)
18958                .await
18959                .unwrap()
18960                .unwrap();
18961            assert_eq!(replies.len(), 1);
18962            // Proof rechecking allowed the relay. Independent tag settlement
18963            // still asks the opener to retry after any same-epoch content change.
18964            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18965            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18966            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18967            let route = rig
18968                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18969                .await;
18970            assert!(rig.live(&route));
18971        }
18972
18973        #[tokio::test]
18974        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18975        {
18976            let mut rig = rig_with_flow_support(false).await;
18977            rig.sync(vec![session(1)]).await;
18978            let route = rig
18979                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18980                .await;
18981            assert!(rig.live(&route));
18982            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18983            let mut record = session(1);
18984            record.attributes.flow_id = Some("flow:7".to_string());
18985            rig.sync(vec![record]).await;
18986            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18987            assert!(rig.live(&unscoped));
18988            assert_eq!(unscoped.stamp(), None);
18989        }
18990
18991        #[tokio::test]
18992        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18993            let mut rig = rig().await;
18994            let mut record = session(1);
18995            record.attributes.flow_id = Some("flow:7".to_string());
18996            rig.sync(vec![record.clone()]).await;
18997            let before = rig.forwarding.published_scope_tag(OWNER, "s");
18998            let mut owner_route = rig
18999                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19000                .await;
19001            let mut carrier_route = rig
19002                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19003                .await;
19004            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19005            rig.sync(vec![record.clone()]).await;
19006            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19007            assert!(rig.live(&owner_route) && owner_route.untouched());
19008            assert!(rig.live(&carrier_route) && carrier_route.untouched());
19009
19010            record.attributes.flow_id = Some("flow:8".to_string());
19011            rig.sync(vec![record]).await;
19012            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19013            let before = before.unwrap();
19014            assert_eq!(after.scope_epoch, before.scope_epoch);
19015            assert!(after.version > before.version);
19016            for route in [&mut owner_route, &mut carrier_route] {
19017                assert!(!rig.live(route));
19018                assert_eq!(
19019                    route.closed_reason(),
19020                    RouteCloseReason::ScopeDelegationChanged
19021                );
19022            }
19023            assert!(rig.live(&unscoped) && unscoped.untouched());
19024            // Each provider also receives a GOODBYE for its drained route;
19025            // consume it before expecting the next route.bind on that sink.
19026            for target in [PLEXUS, OTHER] {
19027                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19028                let goodbye = module_rx
19029                    .try_recv()
19030                    .expect("the provider sees the drain")
19031                    .frame;
19032                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19033                assert!(module_rx.try_recv().is_err());
19034            }
19035            let rebound = rig
19036                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19037                .await;
19038            assert_eq!(
19039                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19040                Some("flow:8")
19041            );
19042        }
19043
19044        /// The owner's sync lands between admission and the module's ack. The
19045        /// open is refused by name, the module's other routes stay up, and the
19046        /// reserved pair is released. Changed content is retryable; an ended
19047        /// scope is not.
19048        #[tokio::test]
19049        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19050            let mut rig = rig().await;
19051            rig.sync(vec![session(1)]).await;
19052            let mut cotenant = rig
19053                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19054                .await;
19055
19056            let mut changed = session(1);
19057            changed.child_owners.push(Principal::Reserved {
19058                module_id: MAGIC.to_string(),
19059            });
19060            let mut ended = None;
19061            for (code, next) in [
19062                (error_codes::SCOPE_CHANGED, vec![changed]),
19063                (error_codes::SCOPE_ENDED, Vec::new()),
19064            ] {
19065                let relayed = rig
19066                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19067                    .await;
19068                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19069                ended = Some(next.is_empty());
19070                rig.sync(next).await;
19071                rig.ack(&relayed).await;
19072                let replies = relayed.task.await.unwrap();
19073                assert_eq!(replies.len(), 1, "{replies:?}");
19074                assert_eq!(parse_error(&replies[0])["code"], code);
19075                assert_eq!(
19076                    subc_protocol::error_codes::is_retryable_route_open(code),
19077                    code == error_codes::SCOPE_CHANGED
19078                );
19079                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19080                // The module is told to drop just the binding it created.
19081                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19082                // Collected, because ending the scope also closes the co-tenant
19083                // route, whose GOODBYE comes first.
19084                let mut goodbyes = Vec::new();
19085                while let Ok(outbound) = plexus_rx.try_recv() {
19086                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19087                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19088                }
19089                assert!(
19090                    goodbyes.contains(&(bind_channel, bind_epoch)),
19091                    "{goodbyes:?}"
19092                );
19093                assert!(rig
19094                    .handler
19095                    .registry
19096                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19097                    .unwrap()
19098                    .is_some());
19099            }
19100            assert_eq!(ended, Some(true));
19101            // The co-tenant stayed up through the change, and closed only when
19102            // the scope ended, by the drain rule rather than by the commit.
19103            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19104        }
19105
19106        /// Each row of the drain table on one set of routes: the owner's, a
19107        /// bare carrier's, and a targeted carrier's to each of its targets.
19108        #[tokio::test]
19109        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19110            struct Case {
19111                name: &'static str,
19112                change: fn(&mut ScopeRecord),
19113                /// Closed routes by index: owner->plexus, aft->plexus,
19114                /// broca->plexus, broca->other.
19115                closed: [Option<RouteCloseReason>; 4],
19116            }
19117            use RouteCloseReason::*;
19118            let cases = [
19119                Case {
19120                    name: "a carrier entry removed",
19121                    change: |r| {
19122                        r.carriers.retain(|c| {
19123                            c.principal
19124                                != Principal::Reserved {
19125                                    module_id: AFT.to_string(),
19126                                }
19127                        })
19128                    },
19129                    closed: [None, Some(ScopeCarrierRemoved), None, None],
19130                },
19131                Case {
19132                    name: "a target removed from a carrier",
19133                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19134                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
19135                },
19136                Case {
19137                    name: "a bare carrier narrowed to targets",
19138                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19139                    closed: [None, Some(ScopeCarrierRemoved), None, None],
19140                },
19141                Case {
19142                    name: "delegates turned off",
19143                    change: |r| r.attributes.delegates = false,
19144                    closed: [Some(ScopeDelegationChanged); 4],
19145                },
19146                Case {
19147                    name: "agent_id changed",
19148                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19149                    closed: [Some(ScopeDelegationChanged); 4],
19150                },
19151                Case {
19152                    name: "a carrier added, child owners changed, the record re-sent",
19153                    change: |r| {
19154                        r.carriers.push(carrier(MAGIC, None));
19155                        r.child_owners.push(Principal::Reserved {
19156                            module_id: MAGIC.to_string(),
19157                        });
19158                    },
19159                    closed: [None; 4],
19160                },
19161                Case {
19162                    name: "a target added",
19163                    change: |r| {
19164                        r.carriers[1]
19165                            .targets
19166                            .as_mut()
19167                            .unwrap()
19168                            .push("third".to_string())
19169                    },
19170                    closed: [None; 4],
19171                },
19172                Case {
19173                    name: "delegates turned on",
19174                    change: |r| r.attributes.delegates = true,
19175                    closed: [None; 4],
19176                },
19177            ];
19178            for case in cases {
19179                let mut rig = rig().await;
19180                rig.sync(vec![session(1)]).await;
19181                let scope = || Some(rig_selector("s", Some(1)));
19182                let mut routes = [
19183                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
19184                    rig.bound(Some(AFT), PLEXUS, scope()).await,
19185                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
19186                    rig.bound(Some(BROCA), OTHER, scope()).await,
19187                ];
19188                let mut record = session(1);
19189                (case.change)(&mut record);
19190                rig.sync(vec![record]).await;
19191                for (index, expected) in case.closed.iter().enumerate() {
19192                    let route = &mut routes[index];
19193                    match expected {
19194                        Some(reason) => {
19195                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
19196                            assert_eq!(
19197                                route.closed_reason(),
19198                                *reason,
19199                                "{}: route {index}",
19200                                case.name
19201                            );
19202                        }
19203                        None => {
19204                            assert!(rig.live(route), "{}: route {index} closed", case.name);
19205                            assert!(
19206                                route.untouched(),
19207                                "{}: route {index} was told something",
19208                                case.name
19209                            );
19210                        }
19211                    }
19212                }
19213            }
19214        }
19215
19216        #[tokio::test]
19217        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19218            // Removed, and replaced by a higher epoch.
19219            for next in [Vec::new(), vec![session(2)]] {
19220                let mut rig = rig().await;
19221                rig.sync(vec![session(1)]).await;
19222                let mut route = rig
19223                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19224                    .await;
19225                rig.sync(next).await;
19226                assert!(!rig.live(&route));
19227                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19228            }
19229
19230            // A child whose parent ends: its routes close as parent-ended, the
19231            // child stays live, and routes under the parent close as ended.
19232            let mut rig = rig().await;
19233            let mut child = session(1);
19234            child.scope_ref = "child".to_string();
19235            child.kind = ScopeKind::Worker;
19236            child.parent = Some(ScopeParent::new(
19237                Principal::Reserved {
19238                    module_id: OWNER.to_string(),
19239                },
19240                "s".to_string(),
19241                1,
19242            ));
19243            rig.sync(vec![session(1), child.clone()]).await;
19244            let mut child_route = rig
19245                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19246                .await;
19247            assert_eq!(
19248                child_route.stamp().unwrap().parent_state,
19249                Some(ParentState::Linked)
19250            );
19251            rig.sync(vec![child]).await;
19252            assert!(!rig.live(&child_route));
19253            assert_eq!(
19254                child_route.closed_reason(),
19255                RouteCloseReason::ScopeParentEnded
19256            );
19257        }
19258
19259        #[tokio::test]
19260        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19261        ) {
19262            let mut rig = rig().await;
19263            rig.sync(vec![session(1)]).await;
19264            let mut route = rig
19265                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19266                .await;
19267            let before = rig.forwarding.published_scope_tag(OWNER, "s");
19268            rig.sync(vec![session(1)]).await;
19269            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19270            assert!(rig.live(&route) && route.untouched());
19271
19272            // A call in flight on the route when another carrier is added. A
19273            // forwarded REQUEST holds one credit on the route's flow until the
19274            // module answers; the router takes it exactly like this.
19275            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19276                .forwarding
19277                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19278                .unwrap()
19279            else {
19280                panic!("the route is bound");
19281            };
19282            binding.flow.acquire_tagged(9, false).await.unwrap();
19283            let mut widened = session(1);
19284            widened.carriers.push(carrier(MAGIC, None));
19285            rig.sync(vec![widened]).await;
19286            assert!(rig.live(&route) && route.untouched());
19287            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19288            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19289            // The call's credit is still held on an open flow, so its answer
19290            // will be delivered: closing the route would have closed the flow.
19291            assert_eq!(binding.flow.in_flight(), 1);
19292            binding
19293                .flow
19294                .acquire_tagged(10, false)
19295                .await
19296                .expect("the flow is still open");
19297        }
19298
19299        /// A swap's superseded endpoint keeps its routes until drained; ending
19300        /// the scope closes them there too.
19301        #[tokio::test]
19302        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19303            let mut rig = rig().await;
19304            rig.sync(vec![session(1)]).await;
19305            let mut on_incumbent = rig
19306                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19307                .await;
19308
19309            // Swap plexus: register a candidate and cut over, leaving the
19310            // incumbent superseded with the route still on it.
19311            let (candidate, _candidate_rx) = wide_ctx(9);
19312            let registration = rig
19313                .handler
19314                .registry
19315                .register_candidate_with_control_ops(
19316                    manifest(PLEXUS, PROTOCOL_VERSION),
19317                    PROTOCOL_VERSION,
19318                    candidate.connection_id,
19319                    module_baseline_control_ops(),
19320                )
19321                .unwrap();
19322            rig.forwarding
19323                .register_candidate_module_connection(
19324                    candidate.connection_id,
19325                    PLEXUS.to_string(),
19326                    PROTOCOL_VERSION,
19327                    manifest_concurrency(&registration.manifest),
19328                    candidate.egress.clone(),
19329                )
19330                .unwrap();
19331            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19332            rig.handler
19333                .registry
19334                .promote_candidate(PLEXUS)
19335                .unwrap()
19336                .unwrap();
19337            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19338
19339            rig.sync(Vec::new()).await;
19340            assert!(!rig.live(&on_incumbent));
19341            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19342            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19343            let goodbye = incumbent_rx
19344                .try_recv()
19345                .expect("the superseded endpoint is told")
19346                .frame;
19347            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19348        }
19349    }
19350}
19351
19352#[cfg(test)]
19353mod concurrency_default_exposure_tests {
19354    use super::*;
19355
19356    fn hello_body(role_json: &str) -> Vec<u8> {
19357        format!(
19358            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19359        )
19360        .into_bytes()
19361    }
19362
19363    fn manifest_from(body: &[u8]) -> ModuleManifest {
19364        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19365        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19366            .expect("manifest parses")
19367    }
19368
19369    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19370
19371    #[test]
19372    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19373        let body = hello_body(&format!(
19374            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19375        ));
19376        let manifest = manifest_from(&body);
19377        // Precondition: serde really resolved it to the default, so the typed
19378        // manifest alone cannot answer the question this probe exists for.
19379        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19380        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19381    }
19382
19383    #[test]
19384    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19385        let body = hello_body(&format!(
19386            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19387        ));
19388        let manifest = manifest_from(&body);
19389        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19390        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19391    }
19392
19393    #[test]
19394    fn non_management_roles_are_never_reported() {
19395        let body = hello_body(
19396            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19397        );
19398        let manifest = manifest_from(&body);
19399        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19400    }
19401}