Skip to main content

subc_daemon/
control.rs

1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6    fmt,
7    net::SocketAddr,
8    path::{Path, PathBuf},
9    sync::{Arc, Mutex, RwLock},
10    time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26    error_codes,
27    manifest::{
28        validate_hello_capability_grammar, validate_hello_confirm_operations,
29        validate_hello_display_name, validate_hello_event_declarations,
30        validate_hello_flow_operations, validate_hello_operation_deferral,
31        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
33        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
34    },
35    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
36    scope::{
37        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
38        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
39        SCOPE_SYNC_OP,
40    },
41    session::{
42        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
43        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
44        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
45        ROLE_VERSIONS_FIELD,
46    },
47    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
48    Priority, RouteTarget, PROTOCOL_VERSION,
49};
50use tokio::time::{timeout_at, Instant};
51use tracing::{debug, info, instrument::WithSubscriber, warn};
52
53use crate::{
54    caller_identity::CallerIdentity,
55    capability_requirements::{
56        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
57        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
58        RequirementStatus, RuntimeModule,
59    },
60    child_roster::DaemonShutdownFlag,
61    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
62    forwarding::{
63        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
64        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
65        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
66    },
67    observability::{
68        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
69    },
70    privileged_prompt::{prompt_text, PrivilegedOperation},
71    provenance::{
72        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
73    },
74    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
75    router::{daemon_error_flags, RouteCtx, RouterError},
76    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
77    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
78    stderr_tail::{CaptureState, TailEntry},
79    supervise::{
80        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
81        SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
82    },
83    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
84};
85
86/// Lowest envelope version this subc build will negotiate.
87///
88/// Module HELLO negotiation is exact: peers must use the daemon's locked
89/// protocol version. Older and newer peers receive `version_unsupported` and
90/// are not registered.
91pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
92
93const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
94const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
95const CAP_PING_PONG: &str = "ping_pong_v1";
96const CAP_SESSION_ATTACH: &str = "session_attach_v1";
97const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
98
99const SUBC_CONTROL_OPS: &[&str] = &[
100    ops::DAEMON_SHUTDOWN_INTENT,
101    ops::ACTIVITY_RECORD,
102    ops::ACTIVITY_QUERY,
103    ops::ACTIVITY_SUBSCRIBE,
104    ops::SERVER_DESCRIBE,
105    ops::CATALOG_LIST,
106    ops::ROUTE_OPEN,
107    ops::ROUTE_POLL,
108    ops::ROUTE_CLOSING,
109    ops::ROUTE_CLOSED,
110    ops::SUPERVISOR_LIST,
111    ops::SUPERVISOR_RESTART,
112    ops::SUPERVISOR_SWAP,
113    ops::SUPERVISOR_RELOAD,
114    ops::SUPERVISOR_RESCAN,
115    ops::SUPERVISOR_RELEASE_RESERVED,
116    ops::SUPERVISOR_SET_ENABLED,
117    ops::SUPERVISOR_HEALTH_PROBE,
118    ops::SUPERVISOR_HEALTH,
119    ops::SUPERVISOR_STDERR_TAIL,
120    ops::SUPERVISOR_TERMINALS,
121    ops::SUPERVISOR_ROUTES,
122    ops::SUPERVISOR_PROVENANCE,
123    ops::SUPERVISOR_SPAWN_SNAPSHOT,
124    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
125];
126
127/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
128/// announced by a capability in `server.describe` instead, and clients check
129/// that capability before sending it. `ck daemon --json` prints `subc_ops`
130/// verbatim and scripts read that output, so a new name there would change it.
131/// A malformed body for one of these is still refused `invalid_control_body`
132/// rather than as an unknown op.
133const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
134
135mod agent_commands;
136mod rescan_checked;
137use rescan_checked::{RescanMode, RescanPlan};
138
139const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
140    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
141    "supervisor.live_roots",
142    "supervisor.agent_commands",
143    SCOPE_SYNC_OP,
144    SCOPE_APPLY_OP,
145    SCOPE_DESCRIBE_OP,
146    subc_protocol::scope::ROUTE_AUTHORIZE_OP,
147    "operator.confirm",
148];
149
150/// Module-originated ops the daemon answers but does not advertise in
151/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
152/// belongs to is incomplete, so no module is told it works before it does.
153const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
154
155const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
156
157/// How long subc waits for a module to ack a relayed route.bind before returning
158/// `module_timeout`. The ack waits on the module's own configure, which for AFT
159/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
160/// and DB-open work — on a cold page cache or a large repo that legitimately
161/// exceeds a couple of seconds. The default is generous because rejecting a VALID
162/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
163/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
164pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
165
166/// How many CONSECUTIVE full-budget relay timeouts against one target module
167/// open that module's bind-relay breaker.
168///
169/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
170/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
171/// so three consecutive full-budget timeouts take ~36s to observe: every client
172/// whose open contributed to opening the breaker had already given up on its
173/// own. That is what makes opening the breaker unable to turn a call that would
174/// have succeeded into a refusal — it can only make an already-failing module
175/// fail faster.
176///
177/// Two would be reachable inside one default deadline. One would convict a
178/// module on a single cold-cache bind, which is exactly the valid-but-slow case
179/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
180pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
181
182/// How long a module's bind-relay breaker stays open before exactly one
183/// `route.open` is let through as a probe.
184///
185/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
186/// re-pays a full-budget stall almost continuously, and the breaker stops being
187/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
188/// deadline: a client that starts retrying after the module recovers has to get
189/// a probe opportunity inside its own deadline, or the breaker converts a
190/// recovered module into a failed call — the failure it exists to prevent,
191/// pointed the other way.
192///
193/// 20s sits between those with room on both sides, and it caps what a wedged
194/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
195/// rather than one per `route.open` per connection. The stall that motivated
196/// this, with its measurements, is written up in
197/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
198/// waited the whole budget out.
199pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
200
201const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
202const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
203
204fn reload_verdict(
205    configured: &Path,
206    spawned_from: Option<&Path>,
207    image: subc_control::RunningImageAgreement,
208) -> PendingReloadVerdict {
209    let path = match spawned_from {
210        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
211            ReloadPathAgreement::Match
212        }
213        Some(spawned_from) => ReloadPathAgreement::Mismatch {
214            configured: configured.to_path_buf(),
215            spawned_from: spawned_from.to_path_buf(),
216        },
217        None => ReloadPathAgreement::Unavailable {
218            reason: if matches!(
219                image,
220                subc_control::RunningImageAgreement::Unavailable {
221                    reason: subc_control::RunningImageUnavailableReason::NotRunning
222                }
223            ) {
224                ReloadPathUnavailableReason::NotRunning
225            } else {
226                ReloadPathUnavailableReason::SpawnedPathUnavailable
227            },
228        },
229    };
230    PendingReloadVerdict::new(path, image)
231}
232
233fn launch_spec_agreement(
234    configured: &crate::supervise::ModuleSpec,
235    running: bool,
236    spawned: Option<&crate::supervise::LaunchSpecDigest>,
237) -> subc_control::LaunchSpecAgreement {
238    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
239    if !running {
240        return Agreement::Unavailable {
241            reason: Reason::NotRunning,
242        };
243    }
244    match spawned {
245        None => Agreement::Unavailable {
246            reason: Reason::SpawnedSpecUnavailable,
247        },
248        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
249            Agreement::Match
250        }
251        Some(_) => Agreement::Mismatch,
252    }
253}
254
255fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
256    if configured == spawned_from {
257        return true;
258    }
259    #[cfg(windows)]
260    {
261        // Windows spawn records the resolved verbatim path (\\?\...), while
262        // configuration keeps its original spelling. Resolve both with the OS
263        // instead of stripping prefixes or folding case, which could conflate
264        // distinct names in case-sensitive directories. Different resolved
265        // paths still differ even if their files have identical content.
266        match (
267            std::fs::canonicalize(configured),
268            std::fs::canonicalize(spawned_from),
269        ) {
270            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
271            _ => false,
272        }
273    }
274    #[cfg(not(windows))]
275    {
276        false
277    }
278}
279
280#[derive(Clone)]
281struct DaemonProvenanceFacts {
282    build: DaemonBuildProvenance,
283    pid: Option<u32>,
284    started_at_ms: Option<u64>,
285    start_clock: Option<crate::clock::StartClock>,
286    executable_path: Option<PathBuf>,
287    executable_identity: Option<SpawnedFileIdentity>,
288    process_start_time: Option<u64>,
289    probe: ExecutableIdentityProbe,
290}
291
292impl Default for DaemonProvenanceFacts {
293    fn default() -> Self {
294        Self {
295            build: DaemonBuildProvenance {
296                build_git_sha: None,
297                build_lock_digest: None,
298            },
299            pid: None,
300            started_at_ms: None,
301            start_clock: None,
302            executable_path: None,
303            executable_identity: None,
304            process_start_time: None,
305            probe: ExecutableIdentityProbe::default(),
306        }
307    }
308}
309
310#[derive(Debug, Clone)]
311struct SupervisorRescanContext {
312    supervisor: Supervisor,
313    config_path: PathBuf,
314    configured_port: Option<u16>,
315    storage_config: Option<crate::daemon_config::StorageConfig>,
316    admission_facts_carrier_module_id: Option<String>,
317    admission_facts_targets: Option<Vec<String>>,
318    scope_authority_owners: Vec<String>,
319    federated_scope_owner: String,
320    privileged_operator_prompt: PrivilegedOperatorPrompt,
321    /// Counts executed rescans that set out to change the supervised modules'
322    /// staged or applied configuration, counted before the first change so a
323    /// rescan that fails partway still counts. With the daemon incarnation it
324    /// names the state a `supervisor.rescan_checked` preview was computed
325    /// against.
326    generation: Arc<std::sync::atomic::AtomicU64>,
327}
328
329/// Refusal labels passed to `observe_route_open_refusal` that mean the target
330/// module is not serving right now, and so open or extend an outage in the
331/// route outage tracker. Every one of them is only reachable after the target
332/// was found in the registry, which is what keeps an arbitrary client-chosen
333/// id from ever creating tracker state.
334///
335/// Deliberately absent: `not_registered` and `removed` (the id may be
336/// anything a client sent, and a removed module is gone on purpose),
337/// `protocol_none` (such a module never serves routes, so nothing is out),
338/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
339/// capability and admission-facts refusals (they refuse the caller, not a
340/// module outage), and `relay_reservation_failed` (its code ranges over
341/// capacity limits as well as a vanished connection). Capacity, breaker,
342/// relay-timeout and module-rejection refusals do not pass through that
343/// function at all; the breaker logs its own transitions.
344///
345/// The two not-serving refusals that bypass that function record themselves
346/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
347/// `required_capability_unprovided` is not tracked: the module itself is up,
348/// and the outage belongs to the missing provider.
349const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
350    "reloading",
351    "supervisor_not_live",
352    "registration_not_active",
353    "no_forwarding_connection",
354    "relay_send_failed",
355];
356
357/// Real channel-0 control handler for subc itself.
358#[derive(Clone)]
359pub struct ControlHandler {
360    registry: Arc<Registry>,
361    activity: Option<crate::activity::ActivityLedger>,
362    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
363    forwarding: Arc<ForwardingTable>,
364    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
365    supervisor: SupervisorHandle,
366    pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
367    agent_declarations: agent_commands::Queues,
368    list_admission: Arc<tokio::sync::Semaphore>,
369    subc_capabilities: Arc<[String]>,
370    /// Daemon-wide route.bind relay budget. Used as the fallback when the
371    /// target module has no per-module override in
372    /// `route_bind_relay_timeouts`.
373    route_bind_relay_timeout: Duration,
374    /// Per-module route.bind relay budget overrides, keyed by module id. When
375    /// `handle_route_open` resolves the deadline for a target module, a
376    /// per-module entry wins over the daemon-wide value above.
377    route_bind_relay_timeouts: BTreeMap<String, Duration>,
378    /// Per-target-module bind-relay breaker state. Shared with the forwarding
379    /// table, which is where a new module connection resets it.
380    route_bind_breakers: RouteBindBreakers,
381    /// Live relay admissions keyed by target module. Shared through the
382    /// forwarding table so cloned or separately built handlers enforce one cap.
383    route_bind_concurrency: RouteBindConcurrency,
384    /// Start and end of each module's not-serving period as seen by
385    /// `route.open`, so an outage gets one line at each edge instead of only
386    /// the per-refusal INFO lines. Taken from the forwarding table, so every
387    /// handler built over one table shares it.
388    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
389    /// Consecutive relay timeouts that open a module's breaker.
390    route_bind_breaker_threshold: u32,
391    /// How long a breaker stays open before one probe is admitted.
392    route_bind_breaker_cooldown: Duration,
393    health_probe_timeout: Duration,
394    /// Central storage policy. When set, each registering module receives its
395    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
396    storage_config: Option<crate::daemon_config::StorageConfig>,
397    /// The machine id established at boot, served on every HELLO_ACK and on
398    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
399    /// changes the file, never this value. `None` serves no id.
400    machine_id: Option<crate::machine_id::MachineId>,
401    admission_facts_carrier_module_id: Option<String>,
402    admission_facts_targets: Option<Vec<String>>,
403    /// Scope records with their sync authorities and tombstones; see
404    /// `crate::scopes`. Shared by clones of this handler, so every connection
405    /// reads and writes one table.
406    scopes: Arc<RwLock<ScopeTable>>,
407    /// Wakes the expiry loop when an accepted scope change may have added a
408    /// deadline, so the loop can wait without ticking while no live scope has
409    /// one. A permit is stored when nothing waits, so no wake is lost.
410    scope_deadline_added: Arc<tokio::sync::Notify>,
411    /// The configured `scope_authority_owners`, kept so a rescan can report a
412    /// changed value as needing a daemon restart; rescan never applies it.
413    scope_authority_owners: Vec<String>,
414    federated_scope_owner: String,
415    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
416    privileged_operator_prompt: PrivilegedOperatorPrompt,
417    /// The local and peer address of each accepted TCP client connection, read
418    /// from the daemon's own socket, so a privileged request can name its caller.
419    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
420    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
421    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
422    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
423    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
424    pub(crate) operator_apps:
425        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
426    /// One launch entry per app, behind a leaf lock: no other lock is taken
427    /// while it is held. Entries are memory-only, so a daemon or
428    /// installed-service restart drops them and a relaunched daemon never
429    /// mistakes an older process for its own launch.
430    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
431    /// Serializes `app.open` launches: the liveness check in the decision and
432    /// the later publish must not interleave across concurrent calls, or two
433    /// calls could both launch and the second publish would orphan the first
434    /// process. Refusals and `already_running` answers share the same lock;
435    /// `app.open` is rare, so the cost is negligible.
436    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
437    // Portable wire samples exercise dispatch without starting an OS process.
438    #[cfg(test)]
439    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
440    #[cfg(test)]
441    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
442    /// Test-only Windows signature and flash hooks. Production builds carry
443    /// neither: with the publisher unset, the production table never reaches
444    /// verification, and the default attention hook only logs.
445    #[allow(dead_code)]
446    #[cfg(windows)]
447    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
448    #[allow(dead_code)]
449    #[cfg(windows)]
450    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
451    /// Test-only tracking fault hook: makes opening the tracking handle on
452    /// the launched pid fail, so a test can prove an untrackable launch is
453    /// stopped rather than left running.
454    #[allow(dead_code)]
455    #[cfg(windows)]
456    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
457    /// Test-only count of Windows spawn attempts, so a test can prove a
458    /// refused launch started no process. Unit tests only.
459    #[cfg(all(test, windows))]
460    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
461    #[cfg(test)]
462    caller_lookup: Option<TestCallerLookup>,
463    /// The launch nonce each module connection presented at HELLO, which is how
464    /// a `scope.sync` is matched to the owner's current launch.
465    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
466    rescan: Option<SupervisorRescanContext>,
467    connected_clients: ConnectedClients,
468    counters: DaemonCounters,
469    /// Why this daemon serves no local endpoint, when binding one failed. `None`
470    /// when the endpoint is served, and also on a handler that was never told.
471    local_endpoint_unavailable: Option<String>,
472    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
473    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
474    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
475    /// their usual severity (see [`log_requirement_events`]). A handler built
476    /// without one has a flag that is never set.
477    daemon_shutdown: DaemonShutdownFlag,
478    daemon_provenance: DaemonProvenanceFacts,
479    #[cfg(test)]
480    control_dispatch_delay: Option<Duration>,
481    #[cfg(test)]
482    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
483    /// Holds `route.open` after the route.bind is queued to the module and
484    /// before it is marked enqueued, so a test can make the module's reply (or
485    /// a client teardown) land exactly inside that window.
486    #[cfg(test)]
487    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
488    /// Holds a HELLO after its registry entry is reserved and before its
489    /// forwarding endpoint is installed, so a test can read the catalog and
490    /// open routes while the module is registered but not yet routable. A
491    /// blocking barrier because the HELLO handler is synchronous; the test
492    /// runs the HELLO on its own thread.
493    #[cfg(test)]
494    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
495}
496
497impl fmt::Debug for ControlHandler {
498    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
499        f.debug_struct("ControlHandler")
500            .field("registry", &self.registry)
501            .field("forwarding", &self.forwarding)
502            .field("process_liveness", &self.process_liveness.is_some())
503            .field("supervisor", &self.supervisor)
504            .field("subc_capabilities", &self.subc_capabilities)
505            .finish()
506    }
507}
508
509struct RouteOpenRequest {
510    target: RouteTarget,
511    identity: BindIdentity,
512    consumer_identity: Option<ConsumerIdentity>,
513    consumer_capabilities: Option<Vec<String>>,
514    role_versions: Option<BTreeMap<String, String>>,
515    busy_refusals: Option<bool>,
516    admission_facts: Option<serde_json::Value>,
517    scope: Option<ScopeSelector>,
518}
519
520#[derive(Clone, Copy)]
521enum RouteOpenPhase {
522    Admission,
523    ProjectRoot,
524    EgressReservation,
525    BindSend,
526    BindAckWait,
527}
528
529/// Store elapsed time for each route-open phase, including attempts that end
530/// in an early refusal or backend error. Admission also includes rechecks and
531/// commit bookkeeping performed between the other timed phases.
532struct RouteOpenTimings {
533    module_id: String,
534    principal: String,
535    admission: Duration,
536    project_root: Duration,
537    egress_reservation: Duration,
538    bind_send: Duration,
539    bind_ack_wait: Duration,
540    phase: Option<RouteOpenPhase>,
541    phase_started_at: Instant,
542}
543
544impl RouteOpenTimings {
545    fn new() -> Self {
546        Self {
547            module_id: String::new(),
548            // Refusals before attestation must not claim a verified principal.
549            principal: "unresolved".to_string(),
550            admission: Duration::ZERO,
551            project_root: Duration::ZERO,
552            egress_reservation: Duration::ZERO,
553            bind_send: Duration::ZERO,
554            bind_ack_wait: Duration::ZERO,
555            phase: Some(RouteOpenPhase::Admission),
556            phase_started_at: Instant::now(),
557        }
558    }
559
560    fn finish(&mut self) {
561        let elapsed = self.phase_started_at.elapsed();
562        match self.phase.take() {
563            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
564            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
565            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
566            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
567            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
568            None => {}
569        }
570    }
571
572    fn enter(&mut self, phase: RouteOpenPhase) {
573        self.finish();
574        self.phase = Some(phase);
575        self.phase_started_at = Instant::now();
576    }
577}
578
579struct RouteBindReservationGuard {
580    forwarding: Arc<ForwardingTable>,
581    endpoint: ModuleEndpointId,
582    relay_corr: u64,
583    armed: bool,
584}
585
586struct ModuleControlRpcGuard {
587    forwarding: Arc<ForwardingTable>,
588    endpoint: ModuleEndpointId,
589    corr: u64,
590    armed: bool,
591}
592
593impl ModuleControlRpcGuard {
594    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
595        Self {
596            forwarding,
597            endpoint,
598            corr,
599            armed: true,
600        }
601    }
602
603    fn disarm(&mut self) {
604        self.armed = false;
605    }
606}
607
608impl Drop for ModuleControlRpcGuard {
609    fn drop(&mut self) {
610        if self.armed {
611            let _ = self
612                .forwarding
613                .cancel_module_control_rpc(self.endpoint, self.corr);
614        }
615    }
616}
617
618impl RouteBindReservationGuard {
619    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
620        Self {
621            forwarding,
622            endpoint,
623            relay_corr,
624            armed: true,
625        }
626    }
627
628    fn release_and_disarm(&mut self) {
629        if !self.armed {
630            return;
631        }
632        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
633            self.endpoint,
634            self.relay_corr,
635            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
636        ) {
637            send_goodbye_target_best_effort(
638                &self.forwarding.counters(),
639                &target,
640                "canceled route.bind",
641            );
642        }
643        self.armed = false;
644    }
645
646    fn disarm(&mut self) {
647        self.armed = false;
648    }
649}
650
651impl Drop for RouteBindReservationGuard {
652    fn drop(&mut self) {
653        self.release_and_disarm();
654    }
655}
656
657/// Per-target-module circuit breaker around the `route.bind` relay.
658///
659/// The connection reader is serial per connection, so a module whose `on_bind`
660/// sits on the ack blocks every LATER frame on the connections that call it,
661/// including calls to unrelated modules. This does not make any module's bind
662/// fast; it stops the daemon paying the full budget again and again for a
663/// condition it has already observed.
664///
665/// State is keyed by TARGET MODULE and shared by every connection: a wedged
666/// module wedges everyone, so what one connection learned should protect the
667/// rest.
668///
669/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
670/// relay to that module has actually timed out, and is removed again when a
671/// relay is accepted or the module reconnects, so it cannot grow with traffic
672/// or with modules that behave.
673///
674/// # Why a `std` mutex here is not the head-of-line defect again
675///
676/// Acquisition never awaits. The critical section is a hash lookup plus a few
677/// integer updates, with no I/O and no `.await` inside it, so a reader task
678/// cannot be descheduled behind it the way it can behind
679/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
680/// primitive, held for the same kind of work, as the refusal counter this very
681/// path already increments.
682///
683/// It is also NOT on the data-plane splice path: only `route.open` and module
684/// registration touch it, so bound-route frames gain no state check and no
685/// contention.
686#[derive(Debug, Clone, Default)]
687pub(crate) struct RouteBindBreakers {
688    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
689}
690
691#[derive(Debug, Clone, Default)]
692pub(crate) struct RouteBindConcurrency {
693    modules: Arc<Mutex<HashMap<String, usize>>>,
694}
695
696struct RouteBindConcurrencyGuard {
697    concurrency: RouteBindConcurrency,
698    module_id: String,
699}
700
701impl RouteBindConcurrency {
702    /// Admit without waiting. Waiting here would move the bind stall from the
703    /// module reply to a semaphore and restore reader head-of-line blocking.
704    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
705        let mut modules = self
706            .modules
707            .lock()
708            .expect("route.bind concurrency mutex poisoned");
709        let in_flight = modules.entry(module_id.to_string()).or_default();
710        if *in_flight >= limit {
711            return Err(*in_flight);
712        }
713        *in_flight += 1;
714        Ok(RouteBindConcurrencyGuard {
715            concurrency: self.clone(),
716            module_id: module_id.to_string(),
717        })
718    }
719}
720
721impl Drop for RouteBindConcurrencyGuard {
722    fn drop(&mut self) {
723        let mut modules = self
724            .concurrency
725            .modules
726            .lock()
727            .expect("route.bind concurrency mutex poisoned");
728        let remove = {
729            let in_flight = modules
730                .get_mut(&self.module_id)
731                .expect("admitted route.bind has a concurrency entry");
732            *in_flight -= 1;
733            *in_flight == 0
734        };
735        if remove {
736            modules.remove(&self.module_id);
737        }
738    }
739}
740
741#[derive(Debug, Default)]
742struct ModuleBreakerState {
743    /// Relay timeouts observed with no accepted relay in between.
744    consecutive_timeouts: u32,
745    /// `Some` while the breaker is open: the instant the cooldown expires and
746    /// the next arrival may probe. `None` means closed.
747    cooldown_until: Option<Instant>,
748    /// A half-open probe has been admitted and has not settled yet. This is
749    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
750    /// that read the cooldown, so concurrent opens arriving at the moment the
751    /// cooldown expires cannot all decide that they are the probe.
752    probe_in_flight: Option<Arc<()>>,
753}
754
755/// What the breaker decided for one `route.open`, before any relay work.
756enum RouteBindAdmission<'a> {
757    Admitted {
758        guard: RouteBindBreakerGuard<'a>,
759        /// This open is the single half-open probe, so the transition is worth
760        /// one log line.
761        probe: bool,
762    },
763    Refused {
764        consecutive_timeouts: u32,
765        /// What is left of the cooldown. Zero when the refusal is because the
766        /// one probe is already in flight rather than because the cooldown has
767        /// not elapsed.
768        retry_in: Duration,
769        probe_in_flight: bool,
770    },
771}
772
773/// An outstanding admission, which must be told how its relay settled.
774///
775/// `Drop` settles it as inconclusive, so an early return between admission and
776/// the relay -- or the whole handler being cancelled when the client
777/// disconnects -- releases a half-open probe slot instead of leaving the
778/// breaker wedged half-open with no further probes.
779struct RouteBindBreakerGuard<'a> {
780    breakers: RouteBindBreakers,
781    module_id: &'a str,
782    probe_token: Option<Arc<()>>,
783    settled: bool,
784}
785
786impl RouteBindBreakerGuard<'_> {
787    /// The module answered within the budget and took the bind. THE ONLY
788    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
789    /// breaker, which is a transition worth logging.
790    fn record_accepted(&mut self) -> bool {
791        self.settled = true;
792        self.breakers.record_accepted(self.module_id)
793    }
794
795    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
796    /// COUNTS TOWARD OPENING.
797    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
798        self.settled = true;
799        self.breakers.record_timeout(
800            self.module_id,
801            self.probe_token.as_ref(),
802            threshold,
803            cooldown,
804        )
805    }
806
807    /// Everything else: the module REJECTED the bind, its connection went away
808    /// mid-relay, or the waiter was cancelled.
809    ///
810    /// None of these is evidence that a module is slow, and each already has
811    /// its own refusal with its own code. A module that rejects a bind in
812    /// microseconds is healthy and must never be convicted for it; a module
813    /// that died has said nothing about the module that replaces it. So these
814    /// neither increment nor reset the count -- they only release a probe slot.
815    fn record_inconclusive(&mut self) {
816        self.settled = true;
817        self.breakers
818            .record_inconclusive(self.module_id, self.probe_token.as_ref());
819    }
820}
821
822impl Drop for RouteBindBreakerGuard<'_> {
823    fn drop(&mut self) {
824        if !self.settled {
825            self.breakers
826                .record_inconclusive(self.module_id, self.probe_token.as_ref());
827        }
828    }
829}
830
831/// The breaker moved to open, reported so the caller can log it outside the
832/// lock. Opening is rare and load-bearing; the refusals that follow are
833/// frequent and are counted rather than logged.
834struct BreakerOpened {
835    consecutive_timeouts: u32,
836    /// True when a failed probe re-opened an already-open breaker, which reads
837    /// very differently in a log from a first opening.
838    reopened_after_probe: bool,
839}
840
841impl RouteBindBreakers {
842    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
843        self.modules
844            .lock()
845            .expect("route.bind breaker mutex poisoned")
846    }
847
848    /// Decide whether this `route.open` may attempt its relay. Takes the map
849    /// lock and nothing else, and never awaits.
850    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
851        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
852            probe: probe_token.is_some(),
853            guard: RouteBindBreakerGuard {
854                breakers: self.clone(),
855                module_id,
856                probe_token,
857                settled: false,
858            },
859        };
860
861        let mut modules = self.lock();
862        let Some(state) = modules.get_mut(module_id) else {
863            return admitted(None);
864        };
865        let Some(cooldown_until) = state.cooldown_until else {
866            return admitted(None);
867        };
868        if state.probe_in_flight.is_some() {
869            return RouteBindAdmission::Refused {
870                consecutive_timeouts: state.consecutive_timeouts,
871                retry_in: Duration::ZERO,
872                probe_in_flight: true,
873            };
874        }
875        let now = Instant::now();
876        if now < cooldown_until {
877            return RouteBindAdmission::Refused {
878                consecutive_timeouts: state.consecutive_timeouts,
879                retry_in: cooldown_until - now,
880                probe_in_flight: false,
881            };
882        }
883        let token = Arc::new(());
884        state.probe_in_flight = Some(Arc::clone(&token));
885        admitted(Some(token))
886    }
887
888    fn record_accepted(&self, module_id: &str) -> bool {
889        self.lock()
890            .remove(module_id)
891            .is_some_and(|state| state.cooldown_until.is_some())
892    }
893
894    fn record_timeout(
895        &self,
896        module_id: &str,
897        probe_token: Option<&Arc<()>>,
898        threshold: u32,
899        cooldown: Duration,
900    ) -> Option<BreakerOpened> {
901        let mut modules = self.lock();
902        let state = modules.entry(module_id.to_string()).or_default();
903        let was_open = state.cooldown_until.is_some();
904        let was_probe = Self::owns_probe(state, probe_token);
905        if was_probe {
906            state.probe_in_flight = None;
907        }
908        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
909        if state.consecutive_timeouts < threshold {
910            return None;
911        }
912        state.cooldown_until = Some(Instant::now() + cooldown);
913        Some(BreakerOpened {
914            consecutive_timeouts: state.consecutive_timeouts,
915            reopened_after_probe: was_open && was_probe,
916        })
917    }
918
919    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
920        // A relay can finish after the breaker was reset or after it opened
921        // again and started a new probe. Only the guard whose token matches the
922        // active probe may release it, so a late relay never frees a newer probe.
923        state
924            .probe_in_flight
925            .as_ref()
926            .zip(token)
927            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
928    }
929
930    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
931        if let Some(state) = self.lock().get_mut(module_id) {
932            if Self::owns_probe(state, probe_token) {
933                state.probe_in_flight = None;
934            }
935        }
936    }
937
938    /// Discard what was learned about a module, because the process it was
939    /// learned about is gone. Returns the discarded count when it was non-zero.
940    ///
941    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
942    /// `module_id` is a configuration identity that outlives any particular
943    /// child; what the breaker observed was the process behind the module
944    /// connection of the moment. When a new connection registers under that id
945    /// the verdict's subject no longer exists, so the verdict is stale by
946    /// construction rather than merely likely to be wrong. Keeping it would
947    /// apply a dead process's record to a live one, which is the same defect
948    /// class this breaker exists to stop the daemon committing.
949    ///
950    /// A half-open probe in flight is discarded with the rest: it was a
951    /// question about the old process.
952    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
953        self.lock()
954            .remove(module_id)
955            .map(|state| state.consecutive_timeouts)
956            .filter(|discarded| *discarded > 0)
957    }
958
959    /// Open breakers, for the `server.describe` counters object. `None` when
960    /// none is open, so the key stays absent rather than present-and-empty.
961    ///
962    /// This is the operator's answer to "is this module refusing instantly or
963    /// is it fine?", which look identical from a client that retries and then
964    /// succeeds.
965    fn open_snapshot(&self) -> Option<serde_json::Value> {
966        let now = Instant::now();
967        let modules = self.lock();
968        let open = modules
969            .iter()
970            .filter_map(|(module_id, state)| {
971                let cooldown_until = state.cooldown_until?;
972                Some((
973                    module_id.clone(),
974                    serde_json::json!({
975                        "consecutive_timeouts": state.consecutive_timeouts,
976                        "cooldown_remaining_ms":
977                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
978                        "probe_in_flight": state.probe_in_flight.is_some(),
979                    }),
980                ))
981            })
982            .collect::<serde_json::Map<String, serde_json::Value>>();
983        (!open.is_empty()).then_some(serde_json::Value::Object(open))
984    }
985}
986
987impl ControlHandler {
988    pub fn new(registry: Arc<Registry>) -> Self {
989        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
990    }
991
992    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
993        let counters = forwarding.counters();
994        // Taken from the forwarding table rather than created here, so that the
995        // breaker a `route.open` consults is the same one a module's
996        // registration resets, however many handlers are built over one table.
997        let route_bind_breakers = forwarding.route_bind_breakers();
998        let route_bind_concurrency = forwarding.route_bind_concurrency();
999        let route_outages = forwarding.route_outages();
1000        Self {
1001            registry,
1002            activity: None,
1003            activity_streams: Arc::new(Mutex::new(HashMap::new())),
1004            forwarding,
1005            process_liveness: None,
1006            supervisor: SupervisorHandle::new(),
1007            resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1008                env!("CARGO_PKG_VERSION").to_owned(),
1009            )),
1010            agent_declarations: agent_commands::Queues::default(),
1011            list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1012            subc_capabilities: Arc::from([
1013                CAP_MANIFEST_REGISTRATION.to_string(),
1014                CAP_CHANNEL_LIFECYCLE.to_string(),
1015                CAP_PING_PONG.to_string(),
1016                CAP_SESSION_ATTACH.to_string(),
1017                CAP_ADMISSION_FACTS_RELAY.to_string(),
1018                CAP_SCOPES_V1.to_string(),
1019                subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1020                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1021                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1022                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1023                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1024                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1025                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1026                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1027                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1028                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1029                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1030                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1031            ]),
1032            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1033            route_bind_relay_timeouts: BTreeMap::new(),
1034            route_bind_breakers,
1035            route_bind_concurrency,
1036            route_outages,
1037            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1038            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1039            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1040            storage_config: None,
1041            machine_id: None,
1042            admission_facts_carrier_module_id: None,
1043            admission_facts_targets: None,
1044            scopes: Arc::new(RwLock::new(ScopeTable::new(
1045                crate::daemon_config::default_scope_authority_owners(),
1046            ))),
1047            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1048            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1049            federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1050            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1051            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1052            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1053            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1054            local_connections: Arc::new(Mutex::new(HashSet::new())),
1055            operator_app_policy: Default::default(),
1056            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1057            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1058            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1059            #[cfg(test)]
1060            app_open_test_platform: None,
1061            #[cfg(test)]
1062            app_open_test_launch: None,
1063            #[cfg(windows)]
1064            windows_verify: None,
1065            #[cfg(windows)]
1066            windows_flash: None,
1067            #[cfg(windows)]
1068            windows_track: None,
1069            #[cfg(all(test, windows))]
1070            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1071            #[cfg(test)]
1072            caller_lookup: None,
1073            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1074            rescan: None,
1075            connected_clients: ConnectedClients::new(),
1076            local_endpoint_unavailable: None,
1077            counters,
1078            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1079            daemon_shutdown: DaemonShutdownFlag::default(),
1080            daemon_provenance: DaemonProvenanceFacts::default(),
1081            #[cfg(test)]
1082            control_dispatch_delay: None,
1083            #[cfg(test)]
1084            provenance_probe_override: None,
1085            #[cfg(test)]
1086            route_bind_relay_sent_hold: None,
1087            #[cfg(test)]
1088            hello_registry_to_forwarding_hold: None,
1089        }
1090    }
1091
1092    /// Set the central storage policy: registering modules then receive their
1093    /// resolved storage descriptor in HELLO_ACK.
1094    pub fn with_storage_config(
1095        mut self,
1096        storage_config: Option<crate::daemon_config::StorageConfig>,
1097    ) -> Self {
1098        self.storage_config = storage_config;
1099        self
1100    }
1101
1102    /// Open the ledger on its own writer thread; an unavailable sink does not
1103    /// prevent the daemon from serving routing or supervisor requests.
1104    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1105        self.activity = Some(crate::activity::ActivityLedger::open(
1106            run_dir.join("activity.jsonl"),
1107            incarnation,
1108        ));
1109        self.supervisor.set_activity(self.activity.clone());
1110        self
1111    }
1112
1113    /// Set the machine id served to every registering module (HELLO_ACK) and on
1114    /// `server.describe`.
1115    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1116        self.machine_id = machine_id;
1117        self
1118    }
1119
1120    /// Configure the exact reserved module and target ids permitted to relay
1121    /// opaque admission facts. Config-file loading validates this authority;
1122    /// this builder keeps the same policy available to embedded test daemons.
1123    pub fn with_admission_facts_config(
1124        mut self,
1125        carrier_module_id: Option<String>,
1126        targets: Option<Vec<String>>,
1127    ) -> Self {
1128        self.admission_facts_carrier_module_id = carrier_module_id;
1129        self.admission_facts_targets = targets;
1130        self
1131    }
1132
1133    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1134    /// Replaces the scope table with an empty one under the new list, so call it
1135    /// while building the handler, before any module can sync.
1136    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1137        self.scopes = Arc::new(RwLock::new(
1138            ScopeTable::new(owners.iter().cloned())
1139                .with_federated_owner(self.federated_scope_owner.clone()),
1140        ));
1141        self.scope_authority_owners = owners;
1142        self
1143    }
1144
1145    /// Set the exclusive federated owner while building the handler, before syncs.
1146    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1147        self.scopes = Arc::new(RwLock::new(
1148            ScopeTable::new(self.scope_authority_owners.iter().cloned())
1149                .with_federated_owner(owner.clone()),
1150        ));
1151        self.federated_scope_owner = owner;
1152        self
1153    }
1154
1155    /// Set the startup policy before installing the rescan context or serving requests.
1156    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1157        self.privileged_operator_prompt = mode;
1158        self
1159    }
1160
1161    /// Override the route.bind relay timeout. Used by tests that assert the
1162    /// timeout path so they don't block on the production-safe default.
1163    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1164        self.route_bind_relay_timeout = timeout;
1165        self
1166    }
1167
1168    /// Install per-module route.bind relay budget overrides. A module id
1169    /// listed here wins over the daemon-wide default set via
1170    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1171    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1172    /// the same `Duration` the bind path will use.
1173    pub fn with_route_bind_relay_timeouts(
1174        mut self,
1175        timeouts: impl IntoIterator<Item = (String, Duration)>,
1176    ) -> Self {
1177        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1178        self
1179    }
1180
1181    /// Resolve the route.bind relay budget for a specific target module id.
1182    /// Per-module overrides win; the daemon-wide value (set via
1183    /// `with_route_bind_relay_timeout` or the built-in default) is the
1184    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1185    /// the same resolution `handle_route_open` will use.
1186    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1187        self.route_bind_relay_timeouts
1188            .get(module_id)
1189            .copied()
1190            .unwrap_or(self.route_bind_relay_timeout)
1191    }
1192
1193    /// Override the per-module bind-relay breaker policy.
1194    ///
1195    /// Used by tests, which cannot spend three production budgets opening a
1196    /// breaker or twenty seconds waiting for its cooldown. The production
1197    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1198    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1199    /// reasoning for the numbers.
1200    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1201        self.route_bind_breaker_threshold = threshold.max(1);
1202        self.route_bind_breaker_cooldown = cooldown;
1203        self
1204    }
1205
1206    #[cfg(test)]
1207    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1208        self.health_probe_timeout = timeout;
1209        self
1210    }
1211
1212    #[cfg(test)]
1213    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1214        self.control_dispatch_delay = Some(delay);
1215        self
1216    }
1217
1218    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1219    /// announce it is parked, once to be released. Build the barrier for two
1220    /// parties, the test being the other one.
1221    #[cfg(test)]
1222    pub(crate) fn with_route_bind_relay_sent_hold(
1223        mut self,
1224        hold: Arc<tokio::sync::Barrier>,
1225    ) -> Self {
1226        self.route_bind_relay_sent_hold = Some(hold);
1227        self
1228    }
1229
1230    /// A HELLO waits on `hold` twice between reserving its registry entry and
1231    /// installing its forwarding endpoint: once to announce it is parked, once
1232    /// to be released. Build the barrier for two parties, the test being the
1233    /// other one, and run the HELLO on a thread of its own.
1234    #[cfg(test)]
1235    pub(crate) fn with_hello_registry_to_forwarding_hold(
1236        mut self,
1237        hold: Arc<std::sync::Barrier>,
1238    ) -> Self {
1239        self.hello_registry_to_forwarding_hold = Some(hold);
1240        self
1241    }
1242
1243    #[cfg(test)]
1244    fn hold_between_registry_and_forwarding(&self) {
1245        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1246            hold.wait();
1247            hold.wait();
1248        }
1249    }
1250
1251    pub fn with_process_liveness(
1252        mut self,
1253        process_liveness: Arc<dyn ModuleProcessLiveness>,
1254    ) -> Self {
1255        self.process_liveness = Some(process_liveness);
1256        self
1257    }
1258
1259    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1260        self.supervisor = supervisor;
1261        self.supervisor.set_activity(self.activity.clone());
1262        self
1263    }
1264
1265    pub fn with_daemon_provenance(
1266        mut self,
1267        pid: u32,
1268        started_at_ms: u64,
1269        executable_path: Option<PathBuf>,
1270        build_git_sha: Option<String>,
1271        build_lock_digest: Option<String>,
1272    ) -> Self {
1273        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1274        let process_start_time = process_start_time(pid);
1275        self.daemon_provenance = DaemonProvenanceFacts {
1276            build: DaemonBuildProvenance {
1277                build_git_sha,
1278                build_lock_digest,
1279            },
1280            pid: Some(pid),
1281            started_at_ms: Some(started_at_ms),
1282            start_clock: None,
1283            executable_path,
1284            executable_identity,
1285            process_start_time,
1286            probe: ExecutableIdentityProbe::default(),
1287        };
1288        self
1289    }
1290
1291    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1292        self.daemon_provenance.start_clock = Some(clock);
1293        self
1294    }
1295
1296    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1297    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1298        self.scopes
1299            .write()
1300            .unwrap_or_else(|poisoned| poisoned.into_inner())
1301            .set_wall_clock(Arc::new(clock));
1302        self
1303    }
1304
1305    #[cfg(test)]
1306    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1307        self.provenance_probe_override = Some(result);
1308        self
1309    }
1310
1311    /// Install the configured module set and its reserved capability bindings.
1312    /// Bindings are configuration-scoped and may point at a provider that has not
1313    /// been installed yet, so this does not require the bound module to exist.
1314    pub fn with_capability_config(
1315        self,
1316        modules: impl IntoIterator<Item = (String, bool)>,
1317        reserved_capabilities: BTreeMap<String, String>,
1318    ) -> Self {
1319        self.capability_evaluator
1320            .configure(modules, reserved_capabilities);
1321        self
1322    }
1323
1324    /// Share the supervisor's daemon-shutdown flag with this handler, so
1325    /// requirement changes after the daemon has begun its announced shutdown
1326    /// are not reported as faults.
1327    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1328        self.daemon_shutdown = flag;
1329        self
1330    }
1331
1332    pub fn with_supervisor_rescan(
1333        mut self,
1334        supervisor: Supervisor,
1335        config_path: impl Into<PathBuf>,
1336        configured_port: Option<u16>,
1337    ) -> Self {
1338        self.rescan = Some(SupervisorRescanContext {
1339            supervisor,
1340            config_path: config_path.into(),
1341            configured_port,
1342            storage_config: self.storage_config.clone(),
1343            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1344            admission_facts_targets: self.admission_facts_targets.clone(),
1345            scope_authority_owners: self.scope_authority_owners.clone(),
1346            federated_scope_owner: self.federated_scope_owner.clone(),
1347            privileged_operator_prompt: self.privileged_operator_prompt,
1348            generation: Arc::default(),
1349        });
1350        self
1351    }
1352
1353    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1354        self.connected_clients = connected_clients;
1355        self
1356    }
1357
1358    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1359    /// an operator can see a bind failure without reading the daemon log.
1360    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1361        self.local_endpoint_unavailable = reason;
1362        self
1363    }
1364
1365    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1366        Arc::clone(&self.forwarding)
1367    }
1368
1369    pub(crate) fn counters(&self) -> DaemonCounters {
1370        self.counters.clone()
1371    }
1372
1373    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1374    /// requirement event without depending on an operator polling a status command.
1375    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1376        tokio::spawn(async move {
1377            loop {
1378                self.capability_evaluator
1379                    .wait_for_change_or_deadline()
1380                    .await;
1381                self.refresh_capability_requirements();
1382            }
1383        });
1384    }
1385
1386    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1387    /// deadline passed during sleep is enforced on the first tick after wake.
1388    /// While no live scope carries a deadline the loop waits for an accepted
1389    /// scope change instead of ticking, so a daemon without deadlines never
1390    /// wakes for expiry.
1391    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1392        tokio::spawn(async move {
1393            loop {
1394                let has_deadlines = self
1395                    .scopes
1396                    .read()
1397                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1398                    .has_deadlines();
1399                if !has_deadlines {
1400                    self.scope_deadline_added.notified().await;
1401                    continue;
1402                }
1403                tokio::time::sleep(Duration::from_secs(1)).await;
1404                if let Err(error) = self.sweep_expired_scopes() {
1405                    warn!(%error, "scope expiry sweep failed");
1406                }
1407            }
1408        });
1409    }
1410
1411    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1412        let mut table = self
1413            .scopes
1414            .write()
1415            .unwrap_or_else(|poisoned| poisoned.into_inner());
1416        let swept = table.sweep_expired();
1417        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1418        drop(table);
1419        self.close_scope_drained_routes(drained);
1420        Ok(())
1421    }
1422
1423    fn publish_scope_changes(
1424        &self,
1425        changes: &[crate::scopes::ScopeTagChange],
1426        expired: &[crate::scopes::ScopeExpired],
1427    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1428        if changes.is_empty() {
1429            return Ok(Vec::new());
1430        }
1431        let drained = self
1432            .forwarding
1433            .publish_scope_changes(changes)
1434            .map_err(RouterError::Forwarding)?;
1435        let mut counts = HashMap::new();
1436        for route in &drained {
1437            *counts
1438                .entry((
1439                    route.scope.owner.as_str(),
1440                    route.scope.scope_ref.as_str(),
1441                    route.scope.tag.scope_epoch,
1442                ))
1443                .or_insert(0usize) += 1;
1444        }
1445        for scope in expired {
1446            let routes_closed = counts
1447                .get(&(
1448                    scope.owner.as_str(),
1449                    scope.scope_ref.as_str(),
1450                    scope.scope_epoch,
1451                ))
1452                .copied()
1453                .unwrap_or(0);
1454            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1455                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1456                routes_closed, "scope expired");
1457        }
1458        Ok(drained)
1459    }
1460
1461    fn runtime_capability_snapshot(
1462        &self,
1463    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1464        let runtime = self
1465            .supervisor
1466            .list()
1467            .into_iter()
1468            .map(|module| {
1469                let status = module.status().map_err(|err| {
1470                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1471                })?;
1472                Ok(RuntimeModule {
1473                    module_id: status.module_id,
1474                    state: status.state,
1475                    enabled: status.enabled,
1476                })
1477            })
1478            .collect::<Result<Vec<_>, RouterError>>()?;
1479        // Include every registration, published or not. The capability
1480        // evaluator counts a module as providing its capabilities as soon as it
1481        // has registered, even before `catalog.list` shows it, so a HELLO in
1482        // progress does not change other modules' required-capability
1483        // verdicts. See `Registry::list_registered_modules`.
1484        let registrations = self.registry.list_registered_modules().map_err(|err| {
1485            RouterError::backend(
1486                0,
1487                0,
1488                format!("failed to list capability registrations: {err}"),
1489            )
1490        })?;
1491        let registrations = registrations
1492            .into_iter()
1493            .map(|registration| RegisteredModule {
1494                module_id: registration.manifest.module_id,
1495                module_version: registration.manifest.module_version,
1496                capabilities: registration.manifest.capabilities,
1497            })
1498            .collect();
1499        Ok((runtime, registrations))
1500    }
1501
1502    /// The capability side effects of a module becoming the active registration
1503    /// for its id: cache its manifest (logging how it differs from the cached
1504    /// one), run the deny census when its declarations call for one, and recompute the
1505    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1506    /// candidate's does not, and the supervisor does it at promotion instead,
1507    /// through [`crate::supervise::SwapPromotionObserver`].
1508    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1509        let cached_registration = RegisteredModule {
1510            module_id: registration.manifest.module_id.clone(),
1511            module_version: registration.manifest.module_version.clone(),
1512            capabilities: registration.manifest.capabilities.clone(),
1513        };
1514        let change = self.capability_evaluator.record_hello(&cached_registration);
1515        log_manifest_change(&cached_registration.module_id, &change);
1516        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1517            self.enforce_capability_denies();
1518        }
1519        self.refresh_capability_requirements();
1520    }
1521
1522    /// Point the shared supervisor handle at this handler for swap promotions.
1523    /// Called wherever a handler is put behind the `Arc` the router serves, so
1524    /// it can be held weakly.
1525    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1526        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1527            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1528        self.supervisor.set_swap_promotion_observer(observer);
1529    }
1530
1531    pub fn refresh_capability_requirements(&self) {
1532        match self.runtime_capability_snapshot() {
1533            Ok((runtime, registrations)) => {
1534                log_requirement_events(
1535                    self.capability_evaluator
1536                        .evaluate_now(&runtime, &registrations),
1537                    &self.daemon_shutdown,
1538                );
1539            }
1540            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1541        }
1542    }
1543
1544    /// Reconcile only live, attested route bindings after a capability deny edge
1545    /// or target claim was added. This is deliberately a control-plane census:
1546    /// the opaque forwarding hot path must not grow a per-frame capability check.
1547    fn enforce_capability_denies(&self) {
1548        let registrations = match self.registry.list_registered_modules() {
1549            Ok(snapshot) => snapshot,
1550            Err(err) => {
1551                warn!(error = %err, "failed to read registrations for capability deny census");
1552                return;
1553            }
1554        };
1555        let manifests = registrations
1556            .into_iter()
1557            .map(|registration| {
1558                (
1559                    registration.manifest.module_id.clone(),
1560                    registration.manifest,
1561                )
1562            })
1563            .collect::<BTreeMap<_, _>>();
1564        let census = match self.forwarding.route_census(None) {
1565            Ok(census) => census,
1566            Err(err) => {
1567                warn!(error = %err, "failed to read route census for capability deny enforcement");
1568                return;
1569            }
1570        };
1571
1572        for (target_module_id, routes) in census {
1573            let Some(target_manifest) = manifests.get(&target_module_id) else {
1574                continue;
1575            };
1576            let mut closed_routes = Vec::new();
1577            let mut goodbyes = Vec::new();
1578            for route in routes {
1579                let Principal::Reserved {
1580                    module_id: opening_module_id,
1581                } = &route.principal
1582                else {
1583                    continue;
1584                };
1585                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1586                    continue;
1587                };
1588                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1589                    continue;
1590                };
1591
1592                match self.forwarding.release_client_route(
1593                    route.goodbye_target.connection_id,
1594                    route.goodbye_target.channel,
1595                    route.goodbye_target.epoch,
1596                ) {
1597                    Ok(RouteRelease::Removed(module_goodbye)) => {
1598                        warn!(
1599                            opening_module_id,
1600                            target_module_id,
1601                            capability,
1602                            "force-closing route because an attested capability deny edge now matches"
1603                        );
1604                        goodbyes.push(module_goodbye);
1605                        goodbyes.push(route.goodbye_target.clone());
1606                        closed_routes.push(route);
1607                    }
1608                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1609                    Err(err) => warn!(
1610                        opening_module_id,
1611                        target_module_id,
1612                        capability,
1613                        error = %err,
1614                        "failed to force-close capability-denied route"
1615                    ),
1616                }
1617            }
1618
1619            if closed_routes.is_empty() {
1620                continue;
1621            }
1622            send_route_control_pushes(
1623                &self.forwarding,
1624                closed_routes,
1625                ClientControlPush::RouteClosed {
1626                    module_id: target_module_id,
1627                    channels: Vec::new(),
1628                    reason: RouteCloseReason::CapabilityDenied,
1629                    drained: false,
1630                    abandoned: 0,
1631                    excluded_subscriptions: 0,
1632                    terminal: Some(false),
1633                },
1634            );
1635            // route.closed records the reason; the route GOODBYE ends the
1636            // client's handle and settles any calls still pending on it.
1637            self.emit_route_goodbyes(goodbyes);
1638        }
1639    }
1640
1641    /// Why a registered module is not accepting new route binds, or `None` when
1642    /// it is. This is the module's effective readiness: its declared readiness
1643    /// first, then every `need: required` capability it declares evaluating to
1644    /// `provided`. `route.open` and `catalog.list` both read it here so the
1645    /// catalog never reports a module routable that `route.open` would refuse.
1646    fn not_ready_reason(
1647        &self,
1648        registration: &crate::registry::ModuleRegistration,
1649    ) -> Option<NotReadyReason> {
1650        if !registration.ready {
1651            return Some(NotReadyReason {
1652                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1653                capability: None,
1654            });
1655        }
1656        self.first_unprovided_required_capability(registration)
1657            .map(|capability| NotReadyReason {
1658                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1659                capability: Some(capability),
1660            })
1661    }
1662
1663    /// The lexicographically first capability this registration declares
1664    /// `need: required` whose evaluator verdict is not `provided`.
1665    ///
1666    /// The verdicts are the capability evaluator's own; nothing here decides
1667    /// what "provided" means. The evaluator counts a capability provided as
1668    /// soon as a module claiming it has REGISTERED, not once that module is
1669    /// ready. That distinction is what keeps two modules that require each
1670    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1671    /// is ready", each would wait for the other to become ready first and
1672    /// neither ever would. Do not tighten it to readiness.
1673    ///
1674    /// A required capability with no verdict at all means this registration's
1675    /// HELLO or catalog.update landed after the last recompute; recompute once
1676    /// rather than let a missing verdict read as either answer. If it is still
1677    /// missing (the recompute itself failed) the capability counts as
1678    /// unprovided: the refusal is retryable, and routing a module whose
1679    /// required provider is unknown is the outcome this check exists to stop.
1680    fn first_unprovided_required_capability(
1681        &self,
1682        registration: &crate::registry::ModuleRegistration,
1683    ) -> Option<String> {
1684        let required = registration
1685            .manifest
1686            .capabilities
1687            .iter()
1688            .flat_map(|declarations| declarations.requires.iter())
1689            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1690            .map(|requirement| requirement.capability.as_str())
1691            .collect::<BTreeSet<_>>();
1692        if required.is_empty() {
1693            return None;
1694        }
1695        let module_id = registration.manifest.module_id.as_str();
1696        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1697        if required
1698            .iter()
1699            .any(|capability| verdict(capability).is_none())
1700        {
1701            self.refresh_capability_requirements();
1702        }
1703        required
1704            .into_iter()
1705            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1706            .map(str::to_string)
1707    }
1708
1709    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1710        self.capability_evaluator
1711            .statuses()
1712            .into_iter()
1713            .map(capability_requirement_status)
1714            .collect()
1715    }
1716
1717    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1718    /// registration-release watch. The signal is what the supervisor waits on
1719    /// before spawning a replacement, so it must only fire once forwarding
1720    /// teardown is also done (see [`Self::cleanup_connection`] /
1721    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1722    /// admitted but its forwarding endpoint could not be installed.
1723    fn deregister_connection(
1724        &self,
1725        connection_id: ConnectionId,
1726        reason: RegistrationEndReason,
1727    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1728        self.registry
1729            .deregister_connection_with_reason(connection_id, reason)
1730    }
1731
1732    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1733        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1734            return None;
1735        }
1736        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1737            parse_client_control_request(&frame.body)
1738        else {
1739            return None;
1740        };
1741        Some(target_module_id(&target).to_string())
1742    }
1743
1744    pub(crate) fn route_open_capacity_refusal(
1745        &self,
1746        ctx: &RouteCtx,
1747        frame: &Frame,
1748        target_module_id: &str,
1749        in_flight: usize,
1750        limit: usize,
1751    ) -> Result<Frame, RouterError> {
1752        // Sent before the request reaches `handle_route_open`, so the caller
1753        // fields come from the raw frame here.
1754        let caller = self.route_open_caller_from_frame(frame);
1755        self.route_open_admission_refusal_frame(
1756            ctx,
1757            &caller,
1758            frame,
1759            target_module_id,
1760            "open_admission_full",
1761            (in_flight, limit),
1762            format!(
1763                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1764            ),
1765        )
1766    }
1767
1768    fn route_open_target_capacity_refusal(
1769        &self,
1770        ctx: &RouteCtx,
1771        caller: &RouteOpenCaller,
1772        frame: &Frame,
1773        target_module_id: &str,
1774        in_flight: usize,
1775    ) -> Result<Frame, RouterError> {
1776        self.route_open_admission_refusal_frame(
1777            ctx,
1778            caller,
1779            frame,
1780            target_module_id,
1781            "target_binds_full",
1782            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1783            format!(
1784                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1785            ),
1786        )
1787    }
1788
1789    /// Admission pressure clears as existing binds settle, so its refusal must
1790    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1791    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1792    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1793    /// cannot currently reach its target; `module_timeout` would falsely claim
1794    /// that a wait expired. A new, cleaner code would be terminal to deployed
1795    /// clients, so it requires a client-tolerance rollout before daemon emission.
1796    #[allow(clippy::too_many_arguments)]
1797    fn route_open_admission_refusal_frame(
1798        &self,
1799        ctx: &RouteCtx,
1800        caller: &RouteOpenCaller,
1801        frame: &Frame,
1802        target_module_id: &str,
1803        reason: &'static str,
1804        (in_flight, limit): (usize, usize),
1805        message: impl Into<String>,
1806    ) -> Result<Frame, RouterError> {
1807        let code = error_codes::TARGET_UNAVAILABLE;
1808        self.counters.increment_route_open_refused(code);
1809        info!(
1810            target: "control",
1811            code,
1812            reason,
1813            module_id = ?target_module_id,
1814            connection_id = ctx.connection_id.get(),
1815            principal = %caller.principal,
1816            scope = %caller.scope,
1817            harness = %caller.harness,
1818            in_flight,
1819            limit,
1820            "route.open refused"
1821        );
1822        control_error_frame(frame, code, message.into())
1823    }
1824
1825    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1826    ///
1827    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1828    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1829    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1830    #[cfg(test)]
1831    pub fn handle_control(
1832        &self,
1833        connection_id: ConnectionId,
1834        frame: Frame,
1835    ) -> Result<Vec<Frame>, RouterError> {
1836        match frame.header.ty {
1837            FrameType::Ping => Ok(vec![pong(&frame)?]),
1838            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1839            FrameType::Goodbye => self.handle_goodbye(connection_id),
1840            ty => Ok(vec![control_error_frame(
1841                &frame,
1842                "unsupported_control_frame",
1843                format!("unsupported channel-0 frame {ty:?}"),
1844            )?]),
1845        }
1846    }
1847
1848    pub async fn handle_control_frame(
1849        &self,
1850        ctx: &RouteCtx,
1851        frame: Frame,
1852    ) -> Result<Vec<Frame>, RouterError> {
1853        self.handle_control_frame_timed(ctx, frame, None).await
1854    }
1855
1856    #[cfg(any(test, feature = "test-support"))]
1857    pub fn with_test_operator_app_policy(
1858        mut self,
1859        policy: crate::operator_apps::TestOperatorAppPolicy,
1860    ) -> Self {
1861        self.operator_app_policy = policy.into();
1862        self
1863    }
1864
1865    /// Inject the Windows signature check for tests. The injected check
1866    /// answers the verified version string or the rejection reason; the
1867    /// daemon enforces the floor itself on the returned version.
1868    #[cfg(all(windows, any(test, feature = "test-support")))]
1869    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1870        self.windows_verify = Some(verify);
1871        self
1872    }
1873
1874    /// Record which pids the daemon would flash for `already_running`.
1875    #[cfg(all(windows, any(test, feature = "test-support")))]
1876    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1877        self.windows_flash = Some(flash);
1878        self
1879    }
1880
1881    /// Inject a tracking failure for tests: makes `track_launched_process`
1882    /// fail so a test can prove a launch that cannot be tracked is stopped,
1883    /// never left running.
1884    #[cfg(all(windows, any(test, feature = "test-support")))]
1885    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1886        self.windows_track = Some(track);
1887        self
1888    }
1889
1890    pub(crate) fn record_captured_peer(
1891        &self,
1892        connection: ConnectionId,
1893        peer: subc_os::local_peer::CapturedPeer,
1894    ) {
1895        self.record_observed_peer(connection, peer.observation().clone());
1896        #[cfg(windows)]
1897        if let Some(stamp) = self.stamp_windows_launch(&peer) {
1898            self.operator_apps
1899                .lock()
1900                .unwrap_or_else(|p| p.into_inner())
1901                .insert(connection, stamp);
1902        }
1903        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1904            self.operator_apps
1905                .lock()
1906                .unwrap_or_else(|p| p.into_inner())
1907                .insert(connection, stamp);
1908        }
1909    }
1910
1911    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1912        self.local_connections
1913            .lock()
1914            .unwrap_or_else(|p| p.into_inner())
1915            .insert(connection);
1916    }
1917
1918    // Keep the connection-level delivery decision shared with route.bind. The
1919    // target's capability opt-in is separate from recognition of the caller.
1920    fn connection_operator_app(
1921        &self,
1922        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1923    ) -> Option<subc_protocol::peer::OperatorApp> {
1924        stamp
1925            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1926            .map(|stamp| stamp.app.clone())
1927    }
1928
1929    fn handle_connection_describe(
1930        &self,
1931        ctx: &RouteCtx,
1932        frame: Frame,
1933    ) -> Result<Vec<Frame>, RouterError> {
1934        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1935            return Ok(vec![control_error_frame(
1936                &frame,
1937                "invalid_control_body",
1938                "malformed connection.describe body",
1939            )?]);
1940        }
1941        let local = self
1942            .local_connections
1943            .lock()
1944            .unwrap_or_else(|p| p.into_inner())
1945            .contains(&ctx.connection_id);
1946        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1947        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1948        Ok(vec![control_response_body_frame(
1949            &frame,
1950            &subc_control::ConnectionDescribeReply::new(local, operator_app),
1951            ops::CONNECTION_DESCRIBE,
1952        )?])
1953    }
1954
1955    pub(crate) fn record_observed_peer(
1956        &self,
1957        connection: ConnectionId,
1958        peer: subc_os::local_peer::ObservedPeer,
1959    ) {
1960        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1961        let peer = match peer {
1962            OsPeer::Unobserved => ObservedPeer::unobserved(),
1963            OsPeer::Observed(peer) => {
1964                let observed = match peer.user {
1965                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1966                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1967                };
1968                observed
1969                    .with_exe(
1970                        peer.exe
1971                            .and_then(|path| path.into_os_string().into_string().ok()),
1972                    )
1973                    .with_signing_id(peer.signing_id)
1974            }
1975        };
1976        self.observed_peers
1977            .lock()
1978            .unwrap_or_else(|p| p.into_inner())
1979            .insert(connection, peer);
1980    }
1981
1982    pub(crate) fn record_caller_endpoints(
1983        &self,
1984        connection: ConnectionId,
1985        endpoints: Option<(SocketAddr, SocketAddr)>,
1986    ) {
1987        let mut callers = self
1988            .caller_endpoints
1989            .lock()
1990            .unwrap_or_else(|p| p.into_inner());
1991        callers.remove(&connection);
1992        if let Some(endpoints) = endpoints {
1993            callers.insert(connection, endpoints);
1994        }
1995    }
1996
1997    #[cfg(test)]
1998    pub(crate) fn with_caller_lookup_for_test(
1999        mut self,
2000        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2001    ) -> Self {
2002        self.caller_lookup = Some(Arc::new(lookup));
2003        self
2004    }
2005
2006    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2007        let endpoints = self
2008            .caller_endpoints
2009            .lock()
2010            .unwrap_or_else(|p| p.into_inner())
2011            .get(&connection)
2012            .copied();
2013        #[cfg(test)]
2014        let lookup = self.caller_lookup.clone();
2015        tokio::task::spawn_blocking(move || {
2016            #[cfg(test)]
2017            if let Some(lookup) = lookup {
2018                return lookup(endpoints);
2019            }
2020            endpoints
2021                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2022                .unwrap_or_default()
2023        })
2024        .await
2025        .unwrap_or_default()
2026    }
2027
2028    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2029        frame.header.channel == 0
2030            && frame.header.ty == FrameType::Request
2031            && self
2032                .forwarding
2033                .module_endpoint_for_connection(ctx.connection_id)
2034                .is_ok_and(|endpoint| endpoint.is_none())
2035            && parse_client_control_request(&frame.body)
2036                .ok()
2037                .is_some_and(|request| privileged_operation(&request).is_some())
2038    }
2039
2040    pub(crate) async fn handle_control_frame_timed(
2041        &self,
2042        ctx: &RouteCtx,
2043        frame: Frame,
2044        dispatch_started_at: Option<StdInstant>,
2045    ) -> Result<Vec<Frame>, RouterError> {
2046        match frame.header.ty {
2047            FrameType::Ping => Ok(vec![pong(&frame)?]),
2048            FrameType::Hello => {
2049                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2050            }
2051            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2052            FrameType::Cancel => {
2053                // Channel-0 cancellation also fences pending declarations;
2054                // successful cancellation has no ordinary reply.
2055                if self
2056                    .agent_declarations
2057                    .cancel(ctx.connection_id, frame.header.corr)
2058                    || self
2059                        .forwarding
2060                        .operator_confirms()
2061                        .cancel(ctx.connection_id, frame.header.corr)
2062                    || self
2063                        .supervisor
2064                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2065                    || self.cancel_activity_subscription(ctx, &frame)?
2066                {
2067                    Ok(Vec::new())
2068                } else {
2069                    Ok(vec![control_error_frame(
2070                        &frame,
2071                        "unknown_subscription",
2072                        "no subscription has this correlation id",
2073                    )?])
2074                }
2075            }
2076            FrameType::Request => {
2077                // This additive operation is not part of the existing exhaustive
2078                // module-control enum. Probe the op before decoding that enum.
2079                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2080                if op
2081                    .as_ref()
2082                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2083                {
2084                    return self.handle_activity_record(ctx, frame).await;
2085                }
2086                if op.as_ref().is_some_and(|probe| {
2087                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2088                }) {
2089                    return self.handle_activity_read(ctx, frame).await;
2090                }
2091                if op
2092                    .as_ref()
2093                    .is_some_and(|probe| probe.op == "operator.confirm")
2094                {
2095                    return self.handle_operator_confirm(ctx, frame);
2096                }
2097                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2098                    return self
2099                        .observe_control_wait(
2100                            ctx,
2101                            None,
2102                            ops::APP_OPEN,
2103                            self.handle_app_open(ctx, frame),
2104                        )
2105                        .await;
2106                }
2107                if op
2108                    .as_ref()
2109                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2110                {
2111                    return self.handle_connection_describe(ctx, frame);
2112                }
2113                if op
2114                    .as_ref()
2115                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2116                {
2117                    return self.handle_scope_served_by(ctx.connection_id, frame);
2118                }
2119                if self
2120                    .forwarding
2121                    .module_endpoint_for_connection(ctx.connection_id)
2122                    .map_err(RouterError::Forwarding)?
2123                    .is_some()
2124                {
2125                    if !is_known_module_request_op(&frame.body) {
2126                        return Ok(vec![control_error_frame(
2127                            &frame,
2128                            "unsupported_control_frame",
2129                            "module-originated channel-0 REQUEST is not supported",
2130                        )?]);
2131                    }
2132                    let request = match parse_module_control_request_from_module(&frame.body) {
2133                        Ok(request) => request,
2134                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2135                            return Ok(vec![control_error_frame(
2136                                &frame,
2137                                "unsupported_control_frame",
2138                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2139                            )?])
2140                        }
2141                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2142                            return Ok(vec![control_error_frame(
2143                                &frame,
2144                                "invalid_control_body",
2145                                format!("malformed module control body: {err}"),
2146                            )?])
2147                        }
2148                    };
2149                    let op = module_control_request_op(&request);
2150                    let corr = frame.header.corr;
2151                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2152                    if let ModuleControlRequestFromModule::AgentCommands {
2153                        started,
2154                        finished,
2155                        known,
2156                    } = request
2157                    {
2158                        return self
2159                            .agent_declarations
2160                            .enqueue(
2161                                self,
2162                                ctx,
2163                                frame,
2164                                crate::resource_totals::Declaration {
2165                                    started,
2166                                    finished,
2167                                    known,
2168                                },
2169                            )
2170                            .await;
2171                    }
2172                    let result =
2173                        self.handle_module_control_request(ctx.connection_id, frame, request);
2174                    log_slow_control_dispatch(
2175                        dispatch_started_at,
2176                        op,
2177                        ctx.connection_id,
2178                        corr,
2179                        None,
2180                    );
2181                    return result;
2182                }
2183
2184                if is_known_module_request_op(&frame.body) {
2185                    let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2186                        .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2187                    let code = if is_agents {
2188                        "not_supervised"
2189                    } else if op
2190                        .as_ref()
2191                        .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2192                    {
2193                        error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2194                    } else {
2195                        "not_registered"
2196                    };
2197                    return Ok(vec![control_error_frame(
2198                        &frame,
2199                        code,
2200                        "module-originated control requests require the module's own registered connection",
2201                    )?]);
2202                }
2203
2204                let request = match parse_client_control_request(&frame.body) {
2205                    Ok(request) => request,
2206                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2207                        return Ok(vec![control_error_frame(
2208                            &frame,
2209                            "unknown_control_op",
2210                            format!("unknown client control op: {err}"),
2211                        )?])
2212                    }
2213                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2214                        return Ok(vec![control_error_frame(
2215                            &frame,
2216                            "invalid_control_body",
2217                            format!("malformed client control body: {err}"),
2218                        )?])
2219                    }
2220                };
2221                let op = client_control_request_op(&request);
2222                let corr = frame.header.corr;
2223                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2224                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2225                #[cfg(test)]
2226                if let Some(delay) = self.control_dispatch_delay {
2227                    tokio::time::sleep(delay).await;
2228                }
2229                let result = self
2230                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2231                    .await;
2232                if let Some(timings) = route_open_timings.as_mut() {
2233                    timings.finish();
2234                }
2235                log_slow_control_dispatch(
2236                    dispatch_started_at,
2237                    op,
2238                    ctx.connection_id,
2239                    corr,
2240                    route_open_timings.as_ref(),
2241                );
2242                result
2243            }
2244            FrameType::Push => {
2245                let Some(endpoint) = self
2246                    .forwarding
2247                    .module_endpoint_for_connection(ctx.connection_id)
2248                    .map_err(RouterError::Forwarding)?
2249                else {
2250                    return Ok(vec![control_error_frame(
2251                        &frame,
2252                        "unsupported_control_frame",
2253                        "client-originated channel-0 PUSH is not supported",
2254                    )?]);
2255                };
2256                self.handle_status_update(endpoint, frame)
2257            }
2258            FrameType::Response | FrameType::Error
2259                if self
2260                    .forwarding
2261                    .module_endpoint_for_connection(ctx.connection_id)
2262                    .map_err(RouterError::Forwarding)?
2263                    .is_some() =>
2264            {
2265                self.handle_module_relay_response(ctx.connection_id, frame)
2266            }
2267            ty => Ok(vec![control_error_frame(
2268                &frame,
2269                "unsupported_control_frame",
2270                format!("unsupported channel-0 frame {ty:?}"),
2271            )?]),
2272        }
2273    }
2274
2275    pub fn cleanup_connection(
2276        &self,
2277        connection_id: ConnectionId,
2278    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2279        self.local_connections
2280            .lock()
2281            .unwrap_or_else(|p| p.into_inner())
2282            .remove(&connection_id);
2283        self.caller_endpoints
2284            .lock()
2285            .unwrap_or_else(|p| p.into_inner())
2286            .remove(&connection_id);
2287        self.build_class_warned
2288            .lock()
2289            .unwrap_or_else(|p| p.into_inner())
2290            .remove(&connection_id);
2291        self.observed_peers
2292            .lock()
2293            .unwrap_or_else(|p| p.into_inner())
2294            .remove(&connection_id);
2295        self.operator_apps
2296            .lock()
2297            .unwrap_or_else(|p| p.into_inner())
2298            .remove(&connection_id);
2299        self.forwarding
2300            .operator_confirms()
2301            .connection_closed(connection_id);
2302        self.cleanup_connection_with_end_reason(
2303            connection_id,
2304            RegistrationEndReason::ConnectionClosed,
2305        )
2306    }
2307
2308    fn cleanup_connection_with_end_reason(
2309        &self,
2310        connection_id: ConnectionId,
2311        requested_reason: RegistrationEndReason,
2312    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2313        self.agent_declarations.close(connection_id);
2314        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2315            self.registry
2316                .get_module_by_connection(connection_id)?
2317                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2318                .and_then(|module| module.registration_end_reason().ok().flatten())
2319                .unwrap_or(requested_reason)
2320        } else {
2321            requested_reason
2322        };
2323        let crash_closed = self
2324            .registry
2325            .get_module_by_connection(connection_id)?
2326            .and_then(|registration| {
2327                self.forwarding
2328                    .module_endpoint_for_connection(connection_id)
2329                    .ok()
2330                    .flatten()
2331                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2332                    .map(|routes| (registration.manifest.module_id, routes))
2333            });
2334        let crash_closed = crash_closed.map(|(module_id, routes)| {
2335            let terminal = match self.supervisor.get(&module_id) {
2336                None => false,
2337                Some(module) => match module.will_recover_after_connection_loss() {
2338                    Ok(will_recover) => !will_recover,
2339                    Err(err) => {
2340                        warn!(
2341                            %module_id,
2342                            error = %err,
2343                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2344                        );
2345                        false
2346                    }
2347                },
2348            };
2349            // The forwarding table gates all providers at the start of daemon
2350            // shutdown, before their connections are closed. An ordinary
2351            // module disconnect still reports crash if that gate is not set.
2352            let (reason, drained) = match self
2353                .forwarding
2354                .daemon_drain_result_for_connection(connection_id)
2355            {
2356                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2357                Ok(None) => (RouteCloseReason::Crash, false),
2358                Err(err) => {
2359                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2360                    (RouteCloseReason::Crash, false)
2361                }
2362            };
2363            (module_id, routes, reason, drained, terminal)
2364        });
2365        let registrations = self.deregister_connection(connection_id, end_reason);
2366        let cleanup = if crash_closed.is_some() {
2367            self.forwarding.cleanup_connection_counted(connection_id)
2368        } else {
2369            // A module connection's teardown needs the count of abandoned
2370            // route.bind relays for its route.closed notice below. Any other
2371            // connection, such as a client's, sends no such notice and needs
2372            // only its routes released, so it uses the route-only wrapper and
2373            // reports zero.
2374            self.forwarding
2375                .cleanup_connection(connection_id)
2376                .map(|released| crate::forwarding::ConnectionCleanup {
2377                    released,
2378                    abandoned_relays: 0,
2379                })
2380        };
2381        // The route.closed push waits for forwarding teardown because only
2382        // teardown knows how many pending route.bind relays it aborted. It still
2383        // goes out before the GOODBYEs for the released routes, and its targets
2384        // were captured above, before teardown removed those routes.
2385        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2386            let abandoned = cleanup
2387                .as_ref()
2388                .map_or(0, |cleanup| cleanup.abandoned_relays);
2389            send_route_control_pushes(
2390                &self.forwarding,
2391                routes,
2392                ClientControlPush::RouteClosed {
2393                    module_id,
2394                    channels: Vec::new(),
2395                    reason,
2396                    drained,
2397                    abandoned,
2398                    excluded_subscriptions: 0,
2399                    terminal: Some(terminal),
2400                },
2401            );
2402        }
2403        if let Ok(cleanup) = cleanup {
2404            self.emit_route_goodbyes(cleanup.released);
2405        }
2406        // Signal the registration-release watch only now that BOTH registry and
2407        // forwarding teardown are done, so a supervisor waiting to spawn a
2408        // replacement never observes release while old routes still exist.
2409        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2410            crate::supervise::notify_registration_release();
2411            self.capability_evaluator.wake_deadline_loop();
2412            self.refresh_capability_requirements();
2413        }
2414        self.supervisor.remove_spawn_subscribers(connection_id);
2415        self.activity_streams
2416            .lock()
2417            .unwrap_or_else(|p| p.into_inner())
2418            .retain(|(id, _), task| {
2419                if *id == connection_id {
2420                    task.abort();
2421                    false
2422                } else {
2423                    true
2424                }
2425            });
2426        // Sync authority dies with its connection, so the owner's next
2427        // connection can take it; the owner's scopes stay as they are.
2428        self.hello_launch_nonces
2429            .lock()
2430            .unwrap_or_else(|poisoned| poisoned.into_inner())
2431            .forget(connection_id);
2432        self.scopes
2433            .write()
2434            .unwrap_or_else(|poisoned| poisoned.into_inner())
2435            .release_connection(connection_id);
2436        registrations
2437    }
2438
2439    pub(crate) fn handle_route_goodbye(
2440        &self,
2441        connection_id: ConnectionId,
2442        route_channel: u16,
2443        route_epoch: u32,
2444    ) -> Result<bool, RouterError> {
2445        debug!(
2446            connection_id = connection_id.get(),
2447            route_channel, route_epoch, "handling route GOODBYE"
2448        );
2449        let RouteRelease::Removed(released_route) = self
2450            .forwarding
2451            .release_client_route(connection_id, route_channel, route_epoch)
2452            .map_err(RouterError::Forwarding)?
2453        else {
2454            return Ok(false);
2455        };
2456        self.emit_route_goodbyes(vec![released_route]);
2457        Ok(true)
2458    }
2459
2460    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2461        for released in released_routes {
2462            let frame = match Frame::build_with_version(
2463                released.negotiated_ver,
2464                FrameType::Goodbye,
2465                control_flags(),
2466                released.channel,
2467                released.epoch,
2468                0,
2469                Vec::new(),
2470            ) {
2471                Ok(frame) => frame,
2472                Err(err) => {
2473                    warn!(
2474                        route_channel = released.channel,
2475                        error = %err,
2476                        "failed to build route GOODBYE frame"
2477                    );
2478                    continue;
2479                }
2480            };
2481            if !released.close_on_delivery_failure() {
2482                crate::forwarding::send_module_route_goodbye(
2483                    &self.counters,
2484                    &released.sink,
2485                    frame,
2486                    released.module_id.as_deref(),
2487                    "client route released",
2488                );
2489                continue;
2490            }
2491            if let Err(err) = released.sink.try_send(frame) {
2492                warn!(
2493                    target_connection_id = released.connection_id.get(),
2494                    route_channel = released.channel,
2495                    error = %err,
2496                    "route GOODBYE was not delivered to client; closing target connection"
2497                );
2498                if self
2499                    .forwarding
2500                    .escalate_client_delivery_failure(
2501                        released.connection_id,
2502                        released.channel,
2503                        released.epoch,
2504                        CloseReason::new(
2505                            "route_goodbye_delivery_failed",
2506                            format!(
2507                                "failed to enqueue route GOODBYE for channel {}: {err}",
2508                                released.channel
2509                            ),
2510                        ),
2511                        crate::forwarding::UndeliveredFrame {
2512                            module_id: released.module_id.as_deref(),
2513                            sink: &released.sink,
2514                        },
2515                    )
2516                    .unwrap_or(false)
2517                {
2518                    self.counters.increment_goodbye_relay_client_failed();
2519                }
2520            }
2521        }
2522    }
2523
2524    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2525    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2526    /// own commit failed after the module had already accepted). Without this, a
2527    /// module that accepts late keeps a binding subc has torn down, so a later
2528    /// frame on that module channel could misdeliver if the channel is reused.
2529    ///
2530    /// Never closes the shared module connection on failure: a dropped notification
2531    /// only wastes a bounded amount of warm module-side state, which the module's
2532    /// own idle reaper reclaims. Only call this once the route.bind relay was
2533    /// actually enqueued to the module — if the relay send itself failed, the
2534    /// module never created a binding and there is nothing to tear down.
2535    fn send_abandoned_route_bind_goodbye(
2536        &self,
2537        module_sink: &crate::FrameSink,
2538        negotiated_ver: u8,
2539        module_channel: u16,
2540        module_epoch: u32,
2541    ) {
2542        let frame = match Frame::build_with_version(
2543            negotiated_ver,
2544            FrameType::Goodbye,
2545            control_flags(),
2546            module_channel,
2547            module_epoch,
2548            0,
2549            Vec::new(),
2550        ) {
2551            Ok(frame) => frame,
2552            Err(err) => {
2553                warn!(
2554                    route_channel = module_channel,
2555                    error = %err,
2556                    "failed to build GOODBYE for abandoned route.bind"
2557                );
2558                return;
2559            }
2560        };
2561        crate::forwarding::send_module_route_goodbye(
2562            &self.counters,
2563            module_sink,
2564            frame,
2565            None,
2566            "abandoned route.bind",
2567        );
2568    }
2569
2570    fn handle_hello(
2571        &self,
2572        connection_id: ConnectionId,
2573        sink: Option<crate::FrameSink>,
2574        frame: Frame,
2575    ) -> Result<Vec<Frame>, RouterError> {
2576        debug!(
2577            connection_id = connection_id.get(),
2578            corr = frame.header.corr,
2579            "handling HELLO"
2580        );
2581        // A module connection has one identity for its entire lifetime. A second
2582        // registration would leave the old registry owner behind while replacing
2583        // its forwarding endpoint and launch nonce.
2584        if self
2585            .registry
2586            .get_module_by_connection(connection_id)
2587            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2588            .is_some()
2589        {
2590            return Ok(vec![control_error_frame(
2591                &frame,
2592                "invalid_hello",
2593                "connection is already registered as a module",
2594            )?]);
2595        }
2596        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2597            Ok(value) => value,
2598            Err(err) => {
2599                return Ok(vec![control_error_frame(
2600                    &frame,
2601                    "invalid_hello",
2602                    format!("malformed HELLO body: {err}"),
2603                )?])
2604            }
2605        };
2606        if let Err(err) = validate_hello_display_name(&hello_value) {
2607            return Ok(vec![control_error_frame(
2608                &frame,
2609                "invalid_manifest",
2610                err.to_string(),
2611            )?]);
2612        }
2613        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2614            return Ok(vec![control_error_frame(
2615                &frame,
2616                "invalid_capability_grammar",
2617                err.to_string(),
2618            )?]);
2619        }
2620        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2621            return Ok(vec![control_error_frame(
2622                &frame,
2623                "invalid_manifest",
2624                err.to_string(),
2625            )?]);
2626        }
2627        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2628            return Ok(vec![control_error_frame(
2629                &frame,
2630                "invalid_event_declaration",
2631                err.to_string(),
2632            )?]);
2633        }
2634        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2635            return Ok(vec![control_error_frame(
2636                &frame,
2637                "invalid_manifest",
2638                err.to_string(),
2639            )?]);
2640        }
2641        if let Some(provenance) = hello_value
2642            .get("manifest")
2643            .and_then(|manifest| manifest.get("provenance"))
2644        {
2645            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2646                return Ok(vec![control_error_frame(
2647                    &frame,
2648                    "invalid_manifest",
2649                    format!("malformed manifest provenance: {err}"),
2650                )?]);
2651            }
2652        }
2653        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2654            let module_id = hello_value["manifest"]["module_id"]
2655                .as_str()
2656                .unwrap_or("<unknown>");
2657            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2658        }
2659        if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
2660            let module_id = hello_value
2661                .get("manifest")
2662                .and_then(|m| m.get("module_id"))
2663                .and_then(serde_json::Value::as_str)
2664                .unwrap_or("unknown");
2665            warn!(module_id, %reason, "dropping operation_deferral from HELLO");
2666        }
2667        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2668            let module_id = hello_value["manifest"]["module_id"]
2669                .as_str()
2670                .unwrap_or("<unknown>");
2671            warn!(module_id, %reason, "dropping flow_operations from HELLO");
2672        }
2673        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2674            let module_id = hello_value["manifest"]["module_id"]
2675                .as_str()
2676                .unwrap_or("<unknown>");
2677            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2678        }
2679        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2680            Ok(hello) => hello,
2681            Err(err) => {
2682                return Ok(vec![control_error_frame(
2683                    &frame,
2684                    "invalid_hello",
2685                    format!("malformed HELLO body: {err}"),
2686                )?])
2687            }
2688        };
2689
2690        if hello.protocol_ver != hello.manifest.protocol_ver {
2691            return Ok(vec![control_error_frame(
2692                &frame,
2693                "invalid_manifest",
2694                format!(
2695                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2696                    hello.protocol_ver, hello.manifest.protocol_ver
2697                ),
2698            )?]);
2699        }
2700
2701        if hello.manifest.module_id.trim().is_empty() {
2702            return Ok(vec![control_error_frame(
2703                &frame,
2704                "invalid_manifest",
2705                "manifest module_id must not be empty",
2706            )?]);
2707        }
2708
2709        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2710            Ok(negotiated_ver) => negotiated_ver,
2711            Err(message) => {
2712                return Ok(vec![control_error_frame(
2713                    &frame,
2714                    "version_unsupported",
2715                    message,
2716                )?])
2717            }
2718        };
2719
2720        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2721        // swap is open for this id, the only HELLO admitted as a second process
2722        // is the one carrying the candidate's launch nonce (the swap token), and
2723        // it registers into the candidate slot rather than being refused as a
2724        // duplicate. Run after the reserved gate, a reserved module's candidate
2725        // would be refused `reserved_module` for presenting a nonce that gate
2726        // does not know. See `SupervisorHandle::swap_hello_admission`.
2727        let swap_admission = self
2728            .supervisor
2729            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2730        if swap_admission == SwapHelloAdmission::Refused {
2731            warn!(
2732                module_id = %hello.manifest.module_id,
2733                connection_id = connection_id.get(),
2734                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2735            );
2736            return Ok(vec![control_error_frame(
2737                &frame,
2738                "swap_token_invalid",
2739                format!(
2740                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2741                    hello.manifest.module_id
2742                ),
2743            )?]);
2744        }
2745        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2746
2747        // A swap candidate proves its identity with the candidate nonce above.
2748        // Other registrations receive this catalog fact only when reserved-module
2749        // policy required and accepted a nonce; unreserved registrations require
2750        // no nonce check.
2751        let verification = if swap_candidate {
2752            ReservedHelloVerification {
2753                nonce_verified: true,
2754                prefix_owner: None,
2755            }
2756        } else {
2757            match self.supervisor.reserved_hello_verification(
2758                &hello.manifest.module_id,
2759                hello.launch_nonce.as_deref(),
2760            ) {
2761                Ok(verified) => verified,
2762                Err(rejection) => {
2763                    let message = match rejection {
2764                        ReservedHelloRejection::Exact { module_id } => format!(
2765                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2766                        ),
2767                        ReservedHelloRejection::Prefix {
2768                            prefix,
2769                            owner_module_id,
2770                        } => format!(
2771                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2772                            hello.manifest.module_id
2773                        ),
2774                    };
2775                    return Ok(vec![control_error_frame(
2776                        &frame,
2777                        "reserved_module",
2778                        message,
2779                    )?]);
2780                }
2781            }
2782        };
2783
2784        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2785            &hello.manifest.module_id,
2786            hello.manifest.capabilities.as_ref(),
2787        );
2788        if let Some(refusal) = reserved_capability_refusals.first() {
2789            let capability = refusal.capability.clone();
2790            let bound_module = refusal.claimants[0].clone();
2791            log_duplicate_claim_events(reserved_capability_refusals);
2792            return Ok(vec![control_error_frame(
2793                &frame,
2794                "reserved_capability",
2795                format!(
2796                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2797                    capability, bound_module, hello.manifest.module_id
2798                ),
2799            )?]);
2800        }
2801
2802        // A connection that already opened client routes must not also register as
2803        // a module: cleanup would then release only one side and leak the other.
2804        if self
2805            .forwarding
2806            .connection_has_client_routes(connection_id)
2807            .map_err(RouterError::Forwarding)?
2808        {
2809            return Ok(vec![control_error_frame(
2810                &frame,
2811                "invalid_hello",
2812                "connection has open client routes and cannot also register as a module",
2813            )?]);
2814        }
2815
2816        // Kept for scope sync authority, which goes only to the connection that
2817        // presented the module's current launch nonce. Recorded before the
2818        // registration is attempted: a connection whose registration then fails
2819        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2820        self.hello_launch_nonces
2821            .lock()
2822            .unwrap_or_else(|poisoned| poisoned.into_inner())
2823            .record(connection_id, hello.launch_nonce.as_deref());
2824        let control_ops = effective_module_control_ops(hello.control_ops);
2825        // Built before anything is registered so an encoding failure leaves no
2826        // registry or forwarding state behind.
2827        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2828        if swap_candidate {
2829            return self.register_swap_candidate(
2830                connection_id,
2831                sink,
2832                &frame,
2833                hello.manifest,
2834                negotiated_ver,
2835                control_ops,
2836                hello_ack,
2837            );
2838        }
2839        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2840        // is refused, but `catalog.list` does not show the module and the
2841        // catalog generation does not move until it is published below, once
2842        // its forwarding endpoint exists. Listing it now would advertise a
2843        // module that `route.open` still refuses for having no forwarding
2844        // connection.
2845        let registration = match self
2846            .registry
2847            .register_unpublished_verified_with_control_ops(
2848                hello.manifest,
2849                negotiated_ver,
2850                connection_id,
2851                control_ops,
2852                verification.nonce_verified,
2853            ) {
2854            Ok(registration) => registration,
2855            Err(RegistryError::DuplicateModuleId { module_id }) => {
2856                return Ok(vec![control_error_frame(
2857                    &frame,
2858                    "duplicate_module_id",
2859                    format!(
2860                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2861                    ),
2862                )?])
2863            }
2864            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2865                return Ok(vec![control_error_frame(
2866                    &frame,
2867                    "invalid_module_id",
2868                    err.to_string(),
2869                )?])
2870            }
2871            Err(err) => {
2872                return Ok(vec![control_error_frame(
2873                    &frame,
2874                    "registry_error",
2875                    err.to_string(),
2876                )?])
2877            }
2878        };
2879        #[cfg(test)]
2880        self.hold_between_registry_and_forwarding();
2881
2882        let reply = if let Some(sink) = sink {
2883            // The forwarding table's module store is also the daemon-to-module
2884            // control-RPC lane, so every HELLO gets a live endpoint even when the
2885            // manifest has no routable provider role. Non-routable modules still
2886            // cannot receive route.bind in production: `handle_route_open` checks
2887            // the registry manifest with `target_has_required_role` before the
2888            // only production call to `begin_route_bind_relay_for` below that
2889            // route.open path. The remaining direct relay callers are unit tests
2890            // and benchmark harnesses that construct forwarding state explicitly.
2891            //
2892            // The HELLO_ACK is queued by the forwarding table itself, before the
2893            // endpoint becomes visible, and is NOT returned as a reply. A module
2894            // reads HELLO_ACK first and exits on anything else; a reply is only
2895            // written after this handler returns, by which time a route.open on
2896            // another connection could already have queued a route.bind request
2897            // for this module ahead of it.
2898            let concurrency = manifest_concurrency(&registration.manifest);
2899            if let Err(err) = self.forwarding.register_module_connection_acked(
2900                connection_id,
2901                registration.manifest.module_id.clone(),
2902                negotiated_ver,
2903                concurrency,
2904                sink,
2905                crate::forwarding::ModuleHelloAck {
2906                    frame: hello_ack,
2907                    verified_prefix_owner: verification.prefix_owner,
2908                },
2909            ) {
2910                // Forwarding registration failed, so there is no forwarding
2911                // state to tear down. Remove the registry entry, which was never
2912                // published and so never listed or counted in the catalog
2913                // generation, and signal the release watch directly.
2914                if matches!(
2915                    self.deregister_connection(
2916                        connection_id,
2917                        RegistrationEndReason::RegistrationFailed,
2918                    ),
2919                    Ok(r) if !r.is_empty()
2920                ) {
2921                    crate::supervise::notify_registration_release();
2922                }
2923                return Ok(vec![control_error_frame(
2924                    &frame,
2925                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2926                        "invalid_hello"
2927                    } else {
2928                        forwarding_error_code(&err)
2929                    },
2930                    err.to_string(),
2931                )?]);
2932            }
2933            Vec::new()
2934        } else {
2935            // No sink means no forwarding endpoint, so nothing can be routed
2936            // ahead of the ack; it goes out as the reply.
2937            vec![hello_ack]
2938        };
2939
2940        // The module is routable now (or, without a sink, as routable as it
2941        // will ever be), so the catalog may list it. Publishing bumps the
2942        // catalog generation in the same registry critical section that makes
2943        // the module listed.
2944        if !self
2945            .registry
2946            .publish_registration(connection_id)
2947            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2948        {
2949            debug!(
2950                module_id = %registration.manifest.module_id,
2951                connection_id = connection_id.get(),
2952                "registration was removed before it could be published"
2953            );
2954        }
2955
2956        // Exposure over assumption: Concurrency's serde default is pinned to the
2957        // pre-field behavior (ModuleManaged), so a management surface that is
2958        // genuinely Serial and just never declared it inherits concurrent
2959        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2960        // turns "no module has been bitten yet" into the checkable claim "no
2961        // module is exposed" -- one read of the boot log instead of a fleet
2962        // audit. Detected from the raw HELLO bytes because the serde default
2963        // deliberately erases the absent/declared distinction from the type.
2964        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2965            info!(
2966                module_id = %registration.manifest.module_id,
2967                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2968            );
2969        }
2970
2971        self.apply_registration_capabilities(&registration);
2972
2973        self.supervisor
2974            .record_activity_registration(&registration.manifest.module_id);
2975
2976        info!(
2977            module_id = %registration.manifest.module_id,
2978            module_version = %registration.manifest.module_version,
2979            negotiated_ver,
2980            routable_provider = manifest_provides_routable_role(&registration.manifest),
2981            connection_id = connection_id.get(),
2982            "module registered"
2983        );
2984
2985        Ok(reply)
2986    }
2987
2988    /// Register a HELLO the swap gate admitted into the candidate slot of the
2989    /// registry and of forwarding, where it is reachable over its own
2990    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2991    /// nothing routes to it until the supervisor cuts over.
2992    ///
2993    /// Registry first (unpublished), then forwarding, then publication, the
2994    /// same order as an ordinary HELLO; a forwarding failure removes the
2995    /// registry entry again. The swap supervisor treats a published, ready
2996    /// candidate as registered and promotes it, and promotion needs the
2997    /// candidate's forwarding endpoint, so the candidate is published only
2998    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2999    /// recompute capability verdicts or run the capability deny census
3000    /// (`enforce_capability_denies`): both are about the module in the active
3001    /// slot, which `route.open` can target. A candidate sits in the candidate
3002    /// slot, so it neither provides capabilities to other modules nor can be
3003    /// routed to until promotion moves it into the active slot.
3004    #[allow(clippy::too_many_arguments)]
3005    fn register_swap_candidate(
3006        &self,
3007        connection_id: ConnectionId,
3008        sink: Option<crate::FrameSink>,
3009        frame: &Frame,
3010        manifest: ModuleManifest,
3011        negotiated_ver: u8,
3012        control_ops: Vec<String>,
3013        hello_ack: Frame,
3014    ) -> Result<Vec<Frame>, RouterError> {
3015        let module_id = manifest.module_id.clone();
3016        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
3017            manifest,
3018            negotiated_ver,
3019            connection_id,
3020            control_ops,
3021        ) {
3022            Ok(registration) => registration,
3023            Err(RegistryError::DuplicateModuleId { module_id }) => {
3024                return Ok(vec![control_error_frame(
3025                    frame,
3026                    "duplicate_module_id",
3027                    format!(
3028                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3029                    ),
3030                )?])
3031            }
3032            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3033                return Ok(vec![control_error_frame(
3034                    frame,
3035                    "invalid_module_id",
3036                    err.to_string(),
3037                )?])
3038            }
3039            Err(err) => {
3040                return Ok(vec![control_error_frame(
3041                    frame,
3042                    "registry_error",
3043                    err.to_string(),
3044                )?])
3045            }
3046        };
3047        #[cfg(test)]
3048        self.hold_between_registry_and_forwarding();
3049        let reply = if let Some(sink) = sink {
3050            // Same ordering as an ordinary HELLO: the forwarding table queues
3051            // the HELLO_ACK before the candidate endpoint is inserted, because
3052            // a module exits if its first frame after HELLO is anything else.
3053            let concurrency = manifest_concurrency(&registration.manifest);
3054            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3055                connection_id,
3056                module_id.clone(),
3057                negotiated_ver,
3058                concurrency,
3059                sink,
3060                hello_ack,
3061            ) {
3062                if matches!(
3063                    self.deregister_connection(
3064                        connection_id,
3065                        RegistrationEndReason::RegistrationFailed,
3066                    ),
3067                    Ok(r) if !r.is_empty()
3068                ) {
3069                    crate::supervise::notify_registration_release();
3070                }
3071                return Ok(vec![control_error_frame(
3072                    frame,
3073                    forwarding_error_code(&err),
3074                    err.to_string(),
3075                )?]);
3076            }
3077            Vec::new()
3078        } else {
3079            vec![hello_ack]
3080        };
3081        if !self
3082            .registry
3083            .publish_registration(connection_id)
3084            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3085        {
3086            debug!(
3087                module_id = %module_id,
3088                connection_id = connection_id.get(),
3089                "swap candidate registration was removed before it could be published"
3090            );
3091        }
3092        self.supervisor.mark_swap_candidate_admitted(&module_id);
3093        info!(
3094            module_id = %module_id,
3095            module_version = %registration.manifest.module_version,
3096            negotiated_ver,
3097            ready = registration.ready,
3098            connection_id = connection_id.get(),
3099            "swap candidate registered; not routable until cutover"
3100        );
3101        Ok(reply)
3102    }
3103
3104    fn build_hello_ack(
3105        &self,
3106        frame: &Frame,
3107        negotiated_ver: u8,
3108        module_id: &str,
3109    ) -> Result<Frame, RouterError> {
3110        let ack = ModuleHelloAckBody {
3111            negotiated_ver,
3112            subc_ops: module_subc_ops(),
3113            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3114            storage: self
3115                .storage_config
3116                .as_ref()
3117                .map(|cfg| cfg.descriptor_for(module_id)),
3118            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3119        };
3120        let body = serde_json::to_vec(&ack).map_err(|err| {
3121            RouterError::backend(
3122                0,
3123                frame.header.corr,
3124                format!("failed to encode HELLO_ACK: {err}"),
3125            )
3126        })?;
3127
3128        Frame::build_with_version(
3129            negotiated_ver,
3130            FrameType::HelloAck,
3131            control_flags(),
3132            0,
3133            0,
3134            frame.header.corr,
3135            body,
3136        )
3137        .map_err(RouterError::FrameBuild)
3138    }
3139
3140    async fn handle_client_control_request(
3141        &self,
3142        ctx: &RouteCtx,
3143        frame: Frame,
3144        request: ClientControlRequest,
3145        route_open_timings: &mut Option<RouteOpenTimings>,
3146    ) -> Result<Vec<Frame>, RouterError> {
3147        if let Some(operation) = privileged_operation(&request) {
3148            let caller = self.resolve_caller(ctx.connection_id).await;
3149            let op = client_control_request_op(&request);
3150            let text = prompt_text(operation, &caller);
3151            match self.privileged_operator_prompt {
3152                PrivilegedOperatorPrompt::Log => {
3153                    info!(target: "subc_daemon::privileged_prompt", operation = op,
3154                        target = ?operation, caller = %caller.description(), summary = %text,
3155                        "privileged request would prompt the operator");
3156                }
3157                PrivilegedOperatorPrompt::Enforce => {
3158                    let key =
3159                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3160                    let stamp = self
3161                        .operator_apps
3162                        .lock()
3163                        .unwrap_or_else(|p| p.into_inner())
3164                        .get(&ctx.connection_id)
3165                        .cloned();
3166                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3167                        &self.operator_app_policy,
3168                        &self.launch_table,
3169                        stamp.as_ref(),
3170                    )
3171                    .then(|| self.shortcut_context(stamp, None, None));
3172                    let outcome = self
3173                        .forwarding
3174                        .operator_confirms()
3175                        .approve_privileged_with_shortcut(
3176                            ctx,
3177                            &frame,
3178                            op,
3179                            text,
3180                            caller.description(),
3181                            key,
3182                            shortcut,
3183                        )
3184                        .await;
3185                    if outcome.code != "confirmed" {
3186                        return Ok(vec![outcome.frame(&frame)]);
3187                    }
3188                }
3189            }
3190        }
3191        match request {
3192            ClientControlRequest::DaemonShutdownIntent { intent } => {
3193                if intent.as_str() != Some("restart") {
3194                    return Ok(vec![control_error_frame(
3195                        &frame,
3196                        "invalid_params",
3197                        "intent must be 'restart'",
3198                    )?]);
3199                }
3200                let response = ClientControlResponse::DaemonShutdownIntent {
3201                    intent: "restart".to_owned(),
3202                    expires_at_ms: self.supervisor.announce_daemon_restart(),
3203                };
3204                Ok(vec![control_response_body_frame(
3205                    &frame,
3206                    &response,
3207                    "ClientControlResponse::DaemonShutdownIntent",
3208                )?])
3209            }
3210            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3211            ClientControlRequest::CatalogList { module_id } => {
3212                self.handle_catalog_list(frame, module_id)
3213            }
3214            ClientControlRequest::RouteOpen {
3215                target,
3216                identity,
3217                consumer_identity,
3218                consumer_capabilities,
3219                role_versions,
3220                busy_refusals,
3221                admission_facts,
3222                scope,
3223                ..
3224            } => {
3225                self.handle_route_open(
3226                    ctx,
3227                    frame,
3228                    RouteOpenRequest {
3229                        target,
3230                        identity,
3231                        consumer_identity,
3232                        consumer_capabilities,
3233                        role_versions,
3234                        busy_refusals,
3235                        admission_facts,
3236                        scope,
3237                    },
3238                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3239                )
3240                .await
3241            }
3242            ClientControlRequest::RoutePoll {
3243                route_channel,
3244                route_epoch,
3245                kind,
3246            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3247            ClientControlRequest::SupervisorList {} => {
3248                let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3249                    self.agent_declarations.refuse(
3250                        self,
3251                        ctx,
3252                        control_error_frame(
3253                            &frame,
3254                            "busy",
3255                            "supervisor list work is at capacity; retry",
3256                        )?,
3257                    );
3258                    return Ok(Vec::new());
3259                };
3260                // Filesystem-backed diagnostics must not hold up this connection's reader.
3261                let handler = self.clone();
3262                let sink = ctx.egress.clone();
3263                tokio::spawn(
3264                    async move {
3265                        let _permit = permit;
3266                        let result = handler.handle_supervisor_list(frame.clone()).await;
3267                        let replies = match result {
3268                            Ok(replies) => replies,
3269                            Err(error) => {
3270                                control_error_frame(&frame, "backend_error", error.to_string())
3271                                    .into_iter()
3272                                    .collect()
3273                            }
3274                        };
3275                        for reply in replies {
3276                            if !matches!(
3277                                tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3278                                    .await,
3279                                Ok(Ok(()))
3280                            ) {
3281                                break;
3282                            }
3283                        }
3284                    }
3285                    .with_current_subscriber(),
3286                );
3287                Ok(Vec::new())
3288            }
3289            ClientControlRequest::SupervisorSpawnSnapshot {} => {
3290                self.handle_supervisor_spawn_snapshot(frame)
3291            }
3292            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3293                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3294            }
3295            ClientControlRequest::SupervisorRestart {
3296                module_id,
3297                drain_timeout_ms,
3298            } => {
3299                self.log_supervisor_request_received(
3300                    ctx,
3301                    frame.header.corr,
3302                    ops::SUPERVISOR_RESTART,
3303                    Some(&module_id),
3304                    None,
3305                )?;
3306                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3307                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3308                    .await
3309            }
3310            ClientControlRequest::SupervisorSwap {
3311                module_id,
3312                ready_timeout_ms,
3313            } => {
3314                self.log_supervisor_request_received(
3315                    ctx,
3316                    frame.header.corr,
3317                    ops::SUPERVISOR_SWAP,
3318                    Some(&module_id),
3319                    None,
3320                )?;
3321                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3322                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3323                    .await
3324            }
3325            ClientControlRequest::SupervisorReload { module_id } => {
3326                self.log_supervisor_request_received(
3327                    ctx,
3328                    frame.header.corr,
3329                    ops::SUPERVISOR_RELOAD,
3330                    Some(&module_id),
3331                    None,
3332                )?;
3333                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3334                self.handle_supervisor_reload(frame, module_id, requester)
3335                    .await
3336            }
3337            ClientControlRequest::SupervisorRescan { preview } => {
3338                if !preview {
3339                    self.log_supervisor_request_received(
3340                        ctx,
3341                        frame.header.corr,
3342                        ops::SUPERVISOR_RESCAN,
3343                        None,
3344                        None,
3345                    )?;
3346                }
3347                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3348                let operation = self.handle_supervisor_rescan(
3349                    frame,
3350                    if preview {
3351                        RescanMode::Preview
3352                    } else {
3353                        RescanMode::Execute
3354                    },
3355                    requester,
3356                );
3357                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3358                    .await
3359            }
3360            ClientControlRequest::SupervisorRescanChecked {
3361                preview,
3362                candidate,
3363                expect,
3364            } => {
3365                self.observe_control_wait(
3366                    ctx,
3367                    None,
3368                    ops::SUPERVISOR_RESCAN,
3369                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3370                )
3371                .await
3372            }
3373            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3374                self.log_supervisor_request_received(
3375                    ctx,
3376                    frame.header.corr,
3377                    ops::SUPERVISOR_RELEASE_RESERVED,
3378                    Some(&module_id),
3379                    None,
3380                )?;
3381                self.handle_supervisor_release_reserved(frame, module_id)
3382                    .await
3383            }
3384            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3385                self.log_supervisor_request_received(
3386                    ctx,
3387                    frame.header.corr,
3388                    ops::SUPERVISOR_SET_ENABLED,
3389                    Some(&module_id),
3390                    Some(enabled),
3391                )?;
3392                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3393                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3394                    .await
3395            }
3396            ClientControlRequest::SupervisorHealthProbe { module_id } => {
3397                self.handle_supervisor_health_probe(ctx, frame, module_id)
3398                    .await
3399            }
3400            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3401            ClientControlRequest::SupervisorRoutes { module_id } => {
3402                self.handle_supervisor_routes(frame, module_id)
3403            }
3404            ClientControlRequest::SupervisorProvenance { module_id } => {
3405                self.handle_supervisor_provenance(ctx, frame, module_id)
3406                    .await
3407            }
3408            ClientControlRequest::SupervisorStderrTail {
3409                module_id,
3410                max_lines,
3411                max_bytes,
3412            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3413            ClientControlRequest::SupervisorTerminals { module_id } => {
3414                self.handle_supervisor_terminals(ctx, frame, module_id)
3415                    .await
3416            }
3417        }
3418    }
3419
3420    fn log_supervisor_request_received(
3421        &self,
3422        ctx: &RouteCtx,
3423        corr: u64,
3424        op: &'static str,
3425        module_id: Option<&str>,
3426        enabled: Option<bool>,
3427    ) -> Result<(), RouterError> {
3428        let caller = self.supervisor_requester(ctx, corr)?;
3429        let caller = principal_label(&caller);
3430        // `Option` fields are recorded only when present, so a request with no
3431        // module id (rescan) or no enabled flag simply omits that field.
3432        info!(
3433            target: "control",
3434            op,
3435            module_id,
3436            enabled,
3437            connection_id = ctx.connection_id.get(),
3438            caller = %caller,
3439            "supervisor request received"
3440        );
3441        Ok(())
3442    }
3443
3444    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3445        Ok(self
3446            .registry
3447            .get_module_by_connection(ctx.connection_id)
3448            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3449            .map(|registration| Principal::Reserved {
3450                module_id: registration.manifest.module_id,
3451            })
3452            .unwrap_or(Principal::Direct))
3453    }
3454
3455    fn handle_module_control_request(
3456        &self,
3457        connection_id: ConnectionId,
3458        frame: Frame,
3459        request: ModuleControlRequestFromModule,
3460    ) -> Result<Vec<Frame>, RouterError> {
3461        match request {
3462            ModuleControlRequestFromModule::RouteAuthorize {
3463                principal,
3464                scope,
3465                target,
3466                expected,
3467            } => self.handle_route_authorize(
3468                connection_id,
3469                frame,
3470                principal,
3471                scope,
3472                target,
3473                expected,
3474            ),
3475            ModuleControlRequestFromModule::CatalogUpdate {
3476                provides,
3477                capabilities,
3478                ready,
3479            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3480            ModuleControlRequestFromModule::LiveRoots {} => {
3481                let registered = self
3482                    .registry
3483                    .get_module_by_connection(connection_id)
3484                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3485                let Some(registration) = registered else {
3486                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3487                };
3488                let response = self
3489                    .forwarding
3490                    .live_roots(&registration.manifest.module_id)
3491                    .map_err(RouterError::Forwarding)?;
3492                Ok(vec![control_response_body_frame(
3493                    &frame,
3494                    &response,
3495                    "ModuleControlResponseToModule::LiveRoots",
3496                )?])
3497            }
3498            ModuleControlRequestFromModule::AgentCommands {
3499                started,
3500                finished,
3501                known,
3502            } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3503            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3504                self.handle_scope_sync(connection_id, frame, generation, scopes)
3505            }
3506            ModuleControlRequestFromModule::ScopeApply {
3507                generation,
3508                upsert,
3509                end,
3510            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3511            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3512                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3513            }
3514            _ => Ok(vec![control_error_frame(
3515                &frame,
3516                "unsupported_control_frame",
3517                "unsupported module control operation",
3518            )?]),
3519        }
3520    }
3521
3522    fn handle_scope_served_by(
3523        &self,
3524        connection_id: ConnectionId,
3525        frame: Frame,
3526    ) -> Result<Vec<Frame>, RouterError> {
3527        let registered = self
3528            .registry
3529            .get_module_by_connection(connection_id)
3530            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3531        let Some(registered) = registered else {
3532            return Ok(vec![control_error_frame(
3533                &frame,
3534                "not_registered",
3535                "scope.served_by requires an active module registration owned by this connection",
3536            )?]);
3537        };
3538        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3539        {
3540            Ok(request) => request,
3541            Err(err) => {
3542                return Ok(vec![control_error_frame(
3543                    &frame,
3544                    "invalid_control_body",
3545                    format!("malformed scope.served_by body: {err}"),
3546                )?])
3547            }
3548        };
3549        let answer = self
3550            .scopes
3551            .read()
3552            .unwrap_or_else(|poisoned| poisoned.into_inner())
3553            .served_by(
3554                &registered.manifest.module_id,
3555                &request.scope_ref,
3556                request.scope_epoch,
3557                &request.module_id,
3558            );
3559        let reply = subc_control::ScopeServedByReply::new(
3560            answer,
3561            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3562        );
3563        Ok(vec![control_response_body_frame(
3564            &frame,
3565            &reply,
3566            "ScopeServedByReply",
3567        )?])
3568    }
3569
3570    fn handle_operator_confirm(
3571        &self,
3572        ctx: &RouteCtx,
3573        frame: Frame,
3574    ) -> Result<Vec<Frame>, RouterError> {
3575        use crate::operator_confirm::{audit, Outcome};
3576        let registration = self
3577            .registry
3578            .get_module_by_connection(ctx.connection_id)
3579            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3580        let Some(registration) = registration else {
3581            let outcome = Outcome::refusal("not_registered");
3582            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3583            return Ok(vec![outcome.frame(&frame)]);
3584        };
3585        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3586            Ok(request) => request,
3587            Err(_) => {
3588                let outcome = Outcome::refusal("invalid_control_body");
3589                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3590                return Ok(vec![outcome.frame(&frame)]);
3591            }
3592        };
3593        let declared = matches!(
3594            request.kind,
3595            Some(subc_protocol::session::OperatorConfirmKind::Operation)
3596        ) && request.operation.as_ref().is_some_and(|operation| {
3597            registration
3598                .manifest
3599                .confirm_operations
3600                .as_ref()
3601                .is_some_and(|operations| operations.contains(operation))
3602        });
3603        let module_id = registration.manifest.module_id;
3604        // Read the launch nonce (under its own lock) before taking the forwarding
3605        // table's lock below: holding forwarding while waiting on another daemon
3606        // lock risks a lock-order deadlock with paths that take them the other way.
3607        let nonce = self
3608            .hello_launch_nonces
3609            .lock()
3610            .unwrap_or_else(|p| p.into_inner())
3611            .nonce(ctx.connection_id)
3612            .map(str::to_owned);
3613        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3614            self.supervisor
3615                .spawned_consumer_authorized(&module_id, nonce)
3616        });
3617        let confirms = self.forwarding.operator_confirms();
3618        self.forwarding
3619            .with_operator_route(
3620                ctx.connection_id,
3621                request.route_channel,
3622                request.route_epoch,
3623                |binding| {
3624                    let shortcut = binding
3625                        .filter(|binding| {
3626                            nonce_proven
3627                                && crate::operator_confirm::valid_summary(&request.summary)
3628                                && declared
3629                                && crate::operator_confirm::ShortcutContext::route_eligible(
3630                                    &self.operator_app_policy,
3631                                    &self.launch_table,
3632                                    binding,
3633                                )
3634                        })
3635                        .map(|binding| {
3636                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3637                                &self.operator_app_policy,
3638                                binding,
3639                            );
3640                            let stamp = if phone.is_some() {
3641                                None
3642                            } else {
3643                                binding.operator_app.clone()
3644                            };
3645                            self.shortcut_context(
3646                                stamp,
3647                                phone,
3648                                Some((request.route_channel, request.route_epoch)),
3649                            )
3650                        });
3651                    confirms.admit_with_shortcut(
3652                        ctx,
3653                        frame,
3654                        module_id,
3655                        nonce_proven,
3656                        request,
3657                        binding,
3658                        shortcut,
3659                    )
3660                },
3661            )
3662            .map_err(RouterError::Forwarding)
3663    }
3664
3665    fn shortcut_context(
3666        &self,
3667        stamp: Option<crate::operator_apps::OperatorAppStamp>,
3668        phone_fingerprint: Option<String>,
3669        route: Option<(u16, u32)>,
3670    ) -> crate::operator_confirm::ShortcutContext {
3671        crate::operator_confirm::ShortcutContext {
3672            ledger: self.activity.clone(),
3673            forwarding: self.forwarding.clone(),
3674            launches: self.launch_table.clone(),
3675            stamp,
3676            phone_fingerprint,
3677            route,
3678            connections: self.operator_apps.clone(),
3679            #[cfg(test)]
3680            hooks: Default::default(),
3681        }
3682    }
3683
3684    /// Explicit test-only phone policy; daemon configuration cannot select it.
3685    #[cfg(any(test, feature = "test-support"))]
3686    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3687        self.operator_app_policy.phone_enabled = enabled;
3688        self
3689    }
3690
3691    /// Resolve folders before taking the scope-table lock: a slow filesystem
3692    /// must not hold up admissions for unrelated scoped routes.
3693    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3694        let mut scopes = ScopeRecords::from(scopes);
3695        for (index, record) in scopes.records.iter_mut().enumerate() {
3696            if let Some(refusal) = Self::canonicalize_workdir(record) {
3697                scopes.refusals.insert(index, refusal);
3698            }
3699        }
3700        scopes
3701    }
3702
3703    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3704        let workdir = record.workdir.as_mut()?;
3705        let path = std::path::Path::new(workdir);
3706        if !path.is_absolute() {
3707            return Some((
3708                error_codes::INVALID_WORKDIR,
3709                "workdir must be absolute".to_string(),
3710            ));
3711        }
3712        match ProjectRootId::from_path_allowing_missing(path) {
3713            Ok(root) => match root.as_path().to_str() {
3714                Some(canonical) => {
3715                    *workdir = canonical.to_string();
3716                    None
3717                }
3718                None => Some((
3719                    error_codes::INVALID_WORKDIR,
3720                    "canonical workdir is not UTF-8".to_string(),
3721                )),
3722            },
3723            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3724        }
3725    }
3726
3727    /// `scope.sync`: the owner is the module registered on this connection.
3728    /// A connection with no registration (every client connection, `direct`
3729    /// included) is refused `not_registered` before the table is consulted.
3730    fn handle_scope_sync(
3731        &self,
3732        connection_id: ConnectionId,
3733        frame: Frame,
3734        generation: u64,
3735        scopes: Vec<ScopeRecord>,
3736    ) -> Result<Vec<Frame>, RouterError> {
3737        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3738    }
3739
3740    fn handle_scope_change(
3741        &self,
3742        connection_id: ConnectionId,
3743        frame: Frame,
3744        generation: u64,
3745        scopes: Vec<ScopeRecord>,
3746        end: Option<Vec<ScopeEnd>>,
3747    ) -> Result<Vec<Frame>, RouterError> {
3748        let is_apply = end.is_some();
3749        let Some(registration) = self
3750            .registry
3751            .get_module_by_connection(connection_id)
3752            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3753        else {
3754            return Ok(vec![control_error_frame(
3755                &frame,
3756                "not_registered",
3757                "scope.sync and scope.apply require an active module registration owned by this connection",
3758            )?]);
3759        };
3760        let owner = registration.manifest.module_id;
3761        let scopes = Self::prepare_scope_records(scopes);
3762        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3763        let is_current_launch = |connection: ConnectionId| {
3764            self.hello_launch_nonces
3765                .lock()
3766                .unwrap_or_else(|poisoned| poisoned.into_inner())
3767                .presented(connection, current_nonce.as_deref())
3768        };
3769        // Lock order is the scope table, then the forwarding table: the new
3770        // tags are published, and the routes the change closes are selected,
3771        // while the scope table is still write-locked, so no admission can read
3772        // a record whose tag is not yet published.
3773        let mut table = self
3774            .scopes
3775            .write()
3776            .unwrap_or_else(|poisoned| poisoned.into_inner());
3777        let outcome = match end {
3778            Some(end) => table.apply(
3779                &owner,
3780                connection_id,
3781                is_current_launch,
3782                generation,
3783                scopes,
3784                end,
3785            ),
3786            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3787        };
3788        let drained = match &outcome {
3789            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3790            Err(_) => Vec::new(),
3791        };
3792        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3793        drop(table);
3794        if has_deadlines {
3795            self.scope_deadline_added.notify_one();
3796        }
3797        match outcome {
3798            Ok(applied) => {
3799                let counts = ScopeOutcomeCounts::of(&applied.results);
3800                info!(
3801                    owner = %owner,
3802                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3803                    generation,
3804                    records = applied.results.len(),
3805                    created = counts.created,
3806                    replaced = counts.replaced,
3807                    updated = counts.updated,
3808                    unchanged = counts.unchanged,
3809                    refused = counts.refused,
3810                    ended = applied.ended.len(),
3811                    tag_changes = applied.tag_changes.len(),
3812                    routes_closed = drained.len(),
3813                    "scope change accepted"
3814                );
3815                // An accepted scope change can still refuse individual records, and the
3816                // owner is the only party that sees the reply. Name them here so
3817                // an operator can tell a refused session from a missing one
3818                // without the owner's logs. Capped so a call that refuses
3819                // thousands cannot flood the log; the count above is complete.
3820                for refused in applied
3821                    .results
3822                    .iter()
3823                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3824                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3825                {
3826                    warn!(
3827                        owner = %owner,
3828                        generation,
3829                        scope_ref = %refused.scope_ref,
3830                        scope_epoch = refused.scope_epoch,
3831                        code = refused.code.as_deref().unwrap_or(""),
3832                        "scope record refused"
3833                    );
3834                }
3835                self.close_scope_drained_routes(drained);
3836                let response = if is_apply {
3837                    ModuleControlResponseToModule::ScopeApply {
3838                        generation,
3839                        results: applied.results,
3840                        end_results: applied.end_results,
3841                        ended: applied.ended,
3842                    }
3843                } else {
3844                    ModuleControlResponseToModule::ScopeSync {
3845                        generation,
3846                        results: applied.results,
3847                        ended: applied.ended,
3848                    }
3849                };
3850                Ok(vec![control_response_body_frame(
3851                    &frame,
3852                    &response,
3853                    "scope change response",
3854                )?])
3855            }
3856            Err(refusal) => {
3857                info!(
3858                    owner = %owner,
3859                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3860                    generation,
3861                    code = refusal.code,
3862                    "scope change refused"
3863                );
3864                Ok(vec![control_error_frame(
3865                    &frame,
3866                    refusal.code,
3867                    refusal.message,
3868                )?])
3869            }
3870        }
3871    }
3872
3873    /// Tell both ends of each route a scope change closed. The module gets a
3874    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3875    /// the client's route handle. The client also gets `route.closed` with the
3876    /// scope reason, one push per module and reason, so it can tell a revoked
3877    /// route from an ordinary close and not reopen it.
3878    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3879        if drained.is_empty() {
3880            return;
3881        }
3882        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3883            BTreeMap::new();
3884        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3885        for route in drained {
3886            warn!(
3887                module_id = %route.module_id,
3888                reason = ?route.reason,
3889                client_connection_id = route.client.connection_id.get(),
3890                route_channel = route.client.channel,
3891                "closing route because its scope changed"
3892            );
3893            pushes
3894                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3895                .or_insert_with(|| (route.reason, Vec::new()))
3896                .1
3897                .push(EndpointRoute {
3898                    goodbye_target: route.client.clone(),
3899                    principal: Principal::Unverified,
3900                    bound_at: Instant::now(),
3901                    draining: false,
3902                    drain_reason: None,
3903                });
3904            goodbyes.push(route.module);
3905            goodbyes.push(route.client);
3906        }
3907        for ((module_id, _), (reason, routes)) in pushes {
3908            send_route_control_pushes(
3909                &self.forwarding,
3910                routes,
3911                ClientControlPush::RouteClosed {
3912                    module_id,
3913                    channels: Vec::new(),
3914                    reason,
3915                    drained: false,
3916                    abandoned: 0,
3917                    excluded_subscriptions: 0,
3918                    terminal: Some(false),
3919                },
3920            );
3921        }
3922        self.emit_route_goodbyes(goodbyes);
3923    }
3924
3925    fn handle_route_authorize(
3926        &self,
3927        connection_id: ConnectionId,
3928        frame: Frame,
3929        principal: Principal,
3930        scope: subc_protocol::scope::RouteAuthorizeScope,
3931        target: String,
3932        expected: subc_protocol::scope::RouteAuthorizeExpected,
3933    ) -> Result<Vec<Frame>, RouterError> {
3934        use crate::scopes::admission::{admit_route, AdmissionMode};
3935        use subc_protocol::scope::{
3936            RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
3937        };
3938        let registry = self
3939            .registry
3940            .route_admission_snapshot()
3941            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3942        let owner_configured = match &scope.owner {
3943            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3944            _ => false,
3945        };
3946        let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
3947        let answer = {
3948            let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
3949            let permitted = registry
3950                .iter()
3951                .find(|r| r.connection_id == connection_id)
3952                .is_some_and(|r| {
3953                    scopes.is_federated_owner(&Principal::Reserved {
3954                        module_id: r.manifest.module_id.clone(),
3955                    }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
3956                        caps.provides
3957                            .iter()
3958                            .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
3959                    })
3960                });
3961            if !permitted {
3962                drop(scopes);
3963                return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
3964                    "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
3965            }
3966            admit_route(
3967                &scopes,
3968                &registry,
3969                &principal,
3970                &target,
3971                Some(&selector),
3972                owner_configured,
3973                AdmissionMode::Authorize(&expected),
3974            )
3975        };
3976        let response = match answer {
3977            Ok(admission) => {
3978                let mut response = RouteAuthorizeResponse::new(
3979                    RouteAuthorizeOutcome::Authorized,
3980                    "route admission authorized",
3981                );
3982                response.content_version = admission.content_version;
3983                response
3984            }
3985            Err(refusal) => refusal.response(),
3986        };
3987        Ok(vec![control_response_body_frame(
3988            &frame,
3989            &response,
3990            "RouteAuthorizeResponse",
3991        )?])
3992    }
3993
3994    /// `scope.describe`: any registered module may read any scope, because a
3995    /// provider must read the scope a route it serves is stamped with.
3996    fn handle_scope_describe(
3997        &self,
3998        connection_id: ConnectionId,
3999        frame: Frame,
4000        owner: Principal,
4001        scope_ref: String,
4002    ) -> Result<Vec<Frame>, RouterError> {
4003        let registered = self
4004            .registry
4005            .get_module_by_connection(connection_id)
4006            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4007        let Some(registered) = registered else {
4008            return Ok(vec![control_error_frame(
4009                &frame,
4010                "not_registered",
4011                "scope.describe requires an active module registration owned by this connection",
4012            )?]);
4013        };
4014        let scopes = self
4015            .scopes
4016            .read()
4017            .unwrap_or_else(|poisoned| poisoned.into_inner());
4018        if scopes.is_federated_owner(&owner)
4019            && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
4020        {
4021            return Ok(vec![control_error_frame(
4022                &frame,
4023                error_codes::SCOPE_FEDERATED_UNSUPPORTED,
4024                "scope.describe of the federated owner requires scope-federated/v1",
4025            )?]);
4026        }
4027        let mut description = scopes.describe(&owner, &scope_ref);
4028        drop(scopes);
4029        if owner
4030            != (Principal::Reserved {
4031                module_id: registered.manifest.module_id.clone(),
4032            })
4033        {
4034            if let Some(facts) = description
4035                .stamp
4036                .as_mut()
4037                .and_then(|stamp| stamp.attributes.federated.as_mut())
4038            {
4039                facts.stamp_required = false;
4040            }
4041        }
4042        // Describe readers also decode strict ScopeAttributes. Attribution is
4043        // safe to omit unless the requesting module declares it understands it.
4044        if !registered
4045            .manifest
4046            .capabilities
4047            .as_ref()
4048            .is_some_and(|caps| {
4049                caps.provides
4050                    .iter()
4051                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4052            })
4053        {
4054            if let Some(stamp) = description.stamp.as_mut() {
4055                stamp.attributes.on_behalf_of_agent = None;
4056            }
4057        }
4058        if !registered
4059            .manifest
4060            .capabilities
4061            .as_ref()
4062            .is_some_and(|caps| {
4063                caps.provides
4064                    .iter()
4065                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
4066            })
4067        {
4068            if let Some(stamp) = description.stamp.as_mut() {
4069                stamp.attributes.flow_version = None;
4070                stamp.attributes.flow_code_hash = None;
4071                stamp.attributes.flow_install_id = None;
4072            }
4073        }
4074        let owner_configured = match &owner {
4075            // Ask whether the owner is configured (`is_configured`), not
4076            // whether it is on the roster (`get(..).is_some()`): a supervised
4077            // module's process can register and describe a scope before the
4078            // supervisor has put it on the roster.
4079            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4080            _ => false,
4081        };
4082        let response = ModuleControlResponseToModule::ScopeDescribe {
4083            status: description.status,
4084            scope_epoch: description.scope_epoch,
4085            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4086            owner_synced: description.owner_synced,
4087            owner_configured,
4088            scope: description.stamp,
4089        };
4090        Ok(vec![control_response_body_frame(
4091            &frame,
4092            &response,
4093            "ModuleControlResponseToModule::ScopeDescribe",
4094        )?])
4095    }
4096
4097    fn handle_catalog_update(
4098        &self,
4099        connection_id: ConnectionId,
4100        frame: Frame,
4101        provides: Vec<ProviderRole>,
4102        capabilities: Option<CapabilityDeclarations>,
4103        ready: Option<bool>,
4104    ) -> Result<Vec<Frame>, RouterError> {
4105        self.refresh_capability_requirements();
4106        let Some(registration) = self
4107            .registry
4108            .get_module_by_connection(connection_id)
4109            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4110        else {
4111            return Ok(vec![control_error_frame(
4112                &frame,
4113                "not_registered",
4114                "catalog.update requires an active module registration owned by this connection",
4115            )?]);
4116        };
4117
4118        if let Some(message) =
4119            catalog_update_frozen_field_message(&registration.manifest, &provides)
4120        {
4121            return Ok(vec![control_error_frame(
4122                &frame,
4123                "catalog_update_frozen_field",
4124                message,
4125            )?]);
4126        }
4127
4128        let mut candidate = registration.manifest.clone();
4129        candidate.provides = provides.clone();
4130        candidate.capabilities = capabilities
4131            .clone()
4132            .or_else(|| registration.manifest.capabilities.clone());
4133        if let Some(rich_ui) = &candidate.rich_ui {
4134            if let Err(err) = rich_ui.validate() {
4135                return Ok(vec![control_error_frame(
4136                    &frame,
4137                    "invalid_manifest",
4138                    err.to_string(),
4139                )?]);
4140            }
4141        }
4142        if let Err(err) = candidate.validate_capability_grammar() {
4143            return Ok(vec![control_error_frame(
4144                &frame,
4145                "invalid_capability_grammar",
4146                err.to_string(),
4147            )?]);
4148        }
4149
4150        // Updates must honor the same reserved owner as initial registration;
4151        // otherwise an empty HELLO could acquire the claim after admission.
4152        let mut conflicts = self
4153            .capability_evaluator
4154            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4155        if let Some(conflict) = conflicts.first() {
4156            let message = format!(
4157                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4158                conflict.capability, conflict.claimants[0], candidate.module_id
4159            );
4160            for conflict in &mut conflicts {
4161                conflict.source = DuplicateClaimSource::CatalogUpdate;
4162            }
4163            log_duplicate_claim_events(conflicts);
4164            return Ok(vec![control_error_frame(
4165                &frame,
4166                "reserved_capability",
4167                message,
4168            )?]);
4169        }
4170
4171        let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4172            Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4173                &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4174                "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4175            )?]),
4176            result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4177        };
4178        if updated.is_none() {
4179            return Ok(vec![control_error_frame(
4180                &frame,
4181                "not_registered",
4182                "catalog.update requires an active module registration owned by this connection",
4183            )?]);
4184        }
4185        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4186            log_duplicate_claim_events(
4187                self.capability_evaluator
4188                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
4189            );
4190        }
4191        if capability_census_trigger(
4192            registration.manifest.capabilities.as_ref(),
4193            updated
4194                .as_ref()
4195                .and_then(|entry| entry.manifest.capabilities.as_ref()),
4196        ) {
4197            self.enforce_capability_denies();
4198        }
4199        self.refresh_capability_requirements();
4200
4201        let response = ModuleControlResponseToModule::CatalogUpdate {};
4202        control_response_body_frame(
4203            &frame,
4204            &response,
4205            "ModuleControlResponseToModule::CatalogUpdate",
4206        )
4207        .map(|frame| vec![frame])
4208    }
4209
4210    pub(crate) fn with_activity_ledger_config(
4211        mut self,
4212        run_dir: &Path,
4213        incarnation: String,
4214        config: crate::activity::ActivityConfig,
4215    ) -> Self {
4216        self.activity = Some(crate::activity::ActivityLedger::open_config(
4217            run_dir.join("activity.jsonl"),
4218            incarnation,
4219            config,
4220        ));
4221        self.supervisor.set_activity(self.activity.clone());
4222        self
4223    }
4224
4225    pub(crate) async fn configure_activity(
4226        &self,
4227        config: crate::activity::ActivityConfig,
4228    ) -> Result<(), &'static str> {
4229        match &self.activity {
4230            Some(ledger) => ledger.configure(config).await,
4231            None => Ok(()),
4232        }
4233    }
4234
4235    fn cancel_activity_subscription(
4236        &self,
4237        ctx: &RouteCtx,
4238        frame: &Frame,
4239    ) -> Result<bool, RouterError> {
4240        let task = self
4241            .activity_streams
4242            .lock()
4243            .unwrap_or_else(|p| p.into_inner())
4244            .remove(&(ctx.connection_id, frame.header.corr));
4245        let Some(task) = task else {
4246            return Ok(false);
4247        };
4248        task.abort();
4249        let end = Frame::build_with_version(
4250            response_version(frame),
4251            FrameType::StreamEnd,
4252            control_flags(),
4253            0,
4254            0,
4255            frame.header.corr,
4256            Vec::new(),
4257        )
4258        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4259        let sink = ctx.egress.clone();
4260        tokio::spawn(async move {
4261            let _ = sink.send(end).await;
4262        });
4263        Ok(true)
4264    }
4265
4266    async fn handle_activity_read(
4267        &self,
4268        ctx: &RouteCtx,
4269        frame: Frame,
4270    ) -> Result<Vec<Frame>, RouterError> {
4271        use subc_control::{
4272            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4273            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4274        };
4275        #[derive(Deserialize)]
4276        struct Identity {
4277            #[serde(default)]
4278            consumer_identity: Option<ConsumerIdentity>,
4279        }
4280        // A registered module connection already identifies its caller. Ignore
4281        // consumer_identity there so a module cannot borrow a mirror's identity.
4282        // Only separate consumer connections use the Direct/nonce trust boundary.
4283        let principal = match self
4284            .forwarding
4285            .module_id_for_connection(ctx.connection_id)
4286            .map_err(RouterError::Forwarding)?
4287        {
4288            Some(module_id) => Ok(Principal::Reserved { module_id }),
4289            None => {
4290                let identity: Identity = match serde_json::from_slice(&frame.body) {
4291                    Ok(identity) => identity,
4292                    Err(_) => {
4293                        return Ok(vec![control_error_frame(
4294                            &frame,
4295                            ACTIVITY_MALFORMED,
4296                            "malformed activity read",
4297                        )?])
4298                    }
4299                };
4300                self.route_open_principal(&frame, identity.consumer_identity)?
4301            }
4302        };
4303        let admitted = matches!(principal, Ok(Principal::Direct))
4304            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4305        if !admitted {
4306            return Ok(vec![control_error_frame(
4307                &frame,
4308                ACTIVITY_READ_FORBIDDEN,
4309                "activity reads require an operator or trusted mirror principal",
4310            )?]);
4311        }
4312        let Some(ledger) = &self.activity else {
4313            return Ok(vec![control_error_frame(
4314                &frame,
4315                ACTIVITY_UNAVAILABLE,
4316                "activity ledger unavailable",
4317            )?]);
4318        };
4319        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4320            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4321        {
4322            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4323                Ok(request) => ledger.query(request).await,
4324                Err(_) => Err(ACTIVITY_MALFORMED),
4325            };
4326            return match result {
4327                Ok(reply) => Ok(vec![control_response_body_frame(
4328                    &frame,
4329                    &reply,
4330                    ops::ACTIVITY_QUERY,
4331                )?]),
4332                Err(code) => Ok(vec![control_error_frame(
4333                    &frame,
4334                    code,
4335                    "activity query refused",
4336                )?]),
4337            };
4338        }
4339        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4340            Ok(request) => request,
4341            Err(_) => {
4342                return Ok(vec![control_error_frame(
4343                    &frame,
4344                    ACTIVITY_MALFORMED,
4345                    "malformed activity subscription",
4346                )?])
4347            }
4348        };
4349        let mut subscription = match ledger.subscribe(request.since).await {
4350            Ok(subscription) => subscription,
4351            Err(code) => {
4352                return Ok(vec![control_error_frame(
4353                    &frame,
4354                    code,
4355                    "activity subscription refused",
4356                )?])
4357            }
4358        };
4359        let key = (ctx.connection_id, frame.header.corr);
4360        let sink = ctx.egress.clone();
4361        let mut streams = self
4362            .activity_streams
4363            .lock()
4364            .unwrap_or_else(|p| p.into_inner());
4365        streams.retain(|_, task| !task.is_finished());
4366        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4367            return Ok(vec![control_error_frame(
4368                &frame,
4369                "duplicate_subscription",
4370                "activity subscription correlation id already in use",
4371            )?]);
4372        }
4373        let task = tokio::spawn(async move {
4374            loop {
4375                let entry = if let Some(entry) = subscription.replay.pop_front() {
4376                    Some(entry)
4377                } else {
4378                    subscription.receiver.recv().await
4379                };
4380                let Some(entry) = entry else {
4381                    break;
4382                };
4383                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4384                    return;
4385                };
4386                let Ok(next) = Frame::build_with_version(
4387                    response_version(&frame),
4388                    FrameType::StreamData,
4389                    control_flags(),
4390                    0,
4391                    0,
4392                    frame.header.corr,
4393                    body,
4394                ) else {
4395                    return;
4396                };
4397                if sink.send(next).await.is_err() {
4398                    return;
4399                }
4400            }
4401            // Dropping a full subscriber's sender drains its accepted entries
4402            // before the terminal error, preserving their order on the wire.
4403            if let Some(code) = subscription.terminal_code() {
4404                if let Ok(end) =
4405                    control_error_frame(&frame, code, "activity subscriber fell behind")
4406                {
4407                    let _ = sink.send(end).await;
4408                }
4409            }
4410        });
4411        streams.insert(key, task.abort_handle());
4412        Ok(Vec::new())
4413    }
4414
4415    async fn handle_activity_record(
4416        &self,
4417        ctx: &RouteCtx,
4418        frame: Frame,
4419    ) -> Result<Vec<Frame>, RouterError> {
4420        use subc_control::{
4421            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4422            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4423        };
4424        let result = async {
4425            let module = self
4426                .forwarding
4427                .module_id_for_connection(ctx.connection_id)
4428                .ok()
4429                .flatten()
4430                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4431            let request: ActivityRecordRequest =
4432                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4433            crate::activity::validate(&module, &request)?;
4434            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4435            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4436            let at_ms = std::time::SystemTime::now()
4437                .duration_since(std::time::UNIX_EPOCH)
4438                .unwrap_or_default()
4439                .as_millis() as u64;
4440            ledger.record(module, request, actor, at_ms).await
4441        }
4442        .await;
4443        match result {
4444            Ok(entry) => Ok(vec![control_response_body_frame(
4445                &frame,
4446                &ActivityRecordResponse { seq: entry.seq },
4447                "activity.record",
4448            )?]),
4449            Err(code) => Ok(vec![control_error_frame(
4450                &frame,
4451                code,
4452                "activity recording refused",
4453            )?]),
4454        }
4455    }
4456
4457    fn activity_actor(
4458        &self,
4459        connection: ConnectionId,
4460        module: &str,
4461        on: Option<&subc_control::ActivityOn>,
4462    ) -> Result<subc_control::ActivityActor, &'static str> {
4463        use crate::forwarding::{DataRoute, DataRouteState};
4464        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4465        let principal = |id: &str| Principal::Reserved {
4466            module_id: id.to_owned(),
4467        };
4468        let mut actor = ActivityActor {
4469            principal: principal(module),
4470            scope_owner: None,
4471            agent_id: None,
4472            run_id: None,
4473            flow_id: None,
4474            project_root: None,
4475            workdir: None,
4476            project_id: None,
4477        };
4478        let scope = match on {
4479            None => return Ok(actor),
4480            Some(ActivityOn::Scope(scope)) => {
4481                let selector = subc_protocol::scope::ScopeSelector::new(
4482                    principal(module),
4483                    scope.scope_ref.clone(),
4484                    Some(scope.epoch),
4485                );
4486                self.scopes
4487                    .read()
4488                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4489                    .attribute(&principal(module), module, &selector, true)
4490                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4491            }
4492            Some(ActivityOn::Route(route)) => {
4493                let DataRoute::Module(DataRouteState::Bound(binding)) = self
4494                    .forwarding
4495                    .lookup_data_route(connection, route.channel, route.epoch)
4496                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4497                else {
4498                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4499                };
4500                if binding.module_id != module {
4501                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4502                }
4503                actor.principal = binding.principal.clone();
4504                actor.project_root = binding
4505                    .project_root
4506                    .as_ref()
4507                    .map(|root| root.as_path().to_path_buf());
4508                let Some(bound) = &binding.scope else {
4509                    return Ok(actor);
4510                };
4511                let owner = principal(&bound.owner);
4512                let selector = subc_protocol::scope::ScopeSelector::new(
4513                    owner.clone(),
4514                    bound.scope_ref.clone(),
4515                    Some(bound.tag.scope_epoch),
4516                );
4517                let admitted = self
4518                    .scopes
4519                    .read()
4520                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4521                    .attribute(&owner, module, &selector, true)
4522                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4523                if admitted.tag != bound.tag {
4524                    return Err(ACTIVITY_ACTOR_UNKNOWN);
4525                }
4526                admitted
4527            }
4528        };
4529        actor.scope_owner = Some(scope.stamp.owner);
4530        actor.agent_id = scope.stamp.attributes.agent_id;
4531        actor.run_id = scope.stamp.attributes.run_id;
4532        actor.flow_id = scope.stamp.attributes.flow_id;
4533        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4534        actor.project_id = scope.stamp.project_id;
4535        Ok(actor)
4536    }
4537
4538    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4539        self.refresh_capability_requirements();
4540        // A bare connection count is ambiguous between many clients holding a
4541        // route each and one client accumulating hundreds, so publish the
4542        // concentration alongside it. Route state is best-effort here: a
4543        // diagnostic endpoint must still answer if the forwarding lock is
4544        // contended.
4545        let mut counters = self.counters.snapshot();
4546        let losses = self.resource_tracker.loss_diagnostics();
4547        if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4548            if let Some(obj) = counters.as_object_mut() {
4549                obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4550            }
4551        }
4552        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4553            obj.insert(
4554                "activity_unparseable_lines".into(),
4555                ledger.read_errors().into(),
4556            );
4557            obj.insert(
4558                "activity_lifecycle_append_failures".into(),
4559                ledger.lifecycle_failures().into(),
4560            );
4561            obj.insert(
4562                "activity_started_digest_skips".into(),
4563                ledger.lifecycle_digest_skips().into(),
4564            );
4565        }
4566        if let (Ok((connections_with_routes, max)), Some(obj)) = (
4567            self.forwarding.client_route_concentration(),
4568            counters.as_object_mut(),
4569        ) {
4570            obj.insert(
4571                "client_connections_with_routes".into(),
4572                connections_with_routes.into(),
4573            );
4574            obj.insert("max_routes_on_one_connection".into(), max.into());
4575        }
4576        // A module that is being fast-refused and a module that is fine look
4577        // identical from a client that retries and succeeds, so name the open
4578        // breakers here. This rides the existing free-form counters object
4579        // rather than a new wire field, so no sibling that deserializes
4580        // `ServerDescribe` has to be rebuilt to keep reading it.
4581        if let (Some(open_breakers), Some(obj)) = (
4582            self.route_bind_breakers.open_snapshot(),
4583            counters.as_object_mut(),
4584        ) {
4585            obj.insert("route_bind_breakers_open".into(), open_breakers);
4586        }
4587        // A daemon whose local endpoint could not be bound keeps serving TCP,
4588        // and the connection file names no endpoint. Name the reason here, in
4589        // the free-form counters object, so an operator reading daemon facts
4590        // sees it without searching the log, and no wire field changes.
4591        if let (Some(reason), Some(obj)) = (
4592            self.local_endpoint_unavailable.clone(),
4593            counters.as_object_mut(),
4594        ) {
4595            obj.insert("local_endpoint_unavailable".into(), reason.into());
4596        }
4597        let response = ClientControlResponse::ServerDescribe {
4598            protocol_ver: PROTOCOL_VERSION,
4599            subc_ops: subc_ops(),
4600            capabilities: self.describe_capabilities(),
4601            connected_clients: self.connected_clients.count(),
4602            counters: Some(counters),
4603            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4604            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4605            capability_requirements: self.capability_requirement_statuses(),
4606            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4607            config_path: self
4608                .rescan
4609                .as_ref()
4610                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4611        };
4612        Ok(vec![control_response_body_frame(
4613            &frame,
4614            &response,
4615            "ClientControlResponse::ServerDescribe",
4616        )?])
4617    }
4618
4619    fn handle_catalog_list(
4620        &self,
4621        frame: Frame,
4622        module_id: Option<String>,
4623    ) -> Result<Vec<Frame>, RouterError> {
4624        let (generation, modules) = self
4625            .registry
4626            .list_modules_with_nonce_verification()
4627            .map_err(|err| {
4628                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4629            })?;
4630        let entries = modules
4631            .into_iter()
4632            .filter(|(registration, _)| {
4633                module_id
4634                    .as_deref()
4635                    .map(|wanted| registration.manifest.module_id == wanted)
4636                    .unwrap_or(true)
4637            })
4638            .map(|(registration, nonce_verified)| {
4639                let not_ready = self.not_ready_reason(&registration);
4640                let roles = registration.manifest.provides;
4641                CatalogEntry::new(
4642                    registration.manifest.module_id,
4643                    roles,
4644                    registration.control_ops,
4645                )
4646                .with_ready(not_ready.is_none())
4647                .with_display_name(registration.manifest.display_name)
4648                .with_not_ready(not_ready)
4649                .with_module_version(Some(registration.manifest.module_version))
4650                .with_provenance(registration.manifest.provenance)
4651                .with_capabilities(registration.manifest.capabilities)
4652                .with_self_signals(registration.manifest.self_signals)
4653                .with_events(registration.manifest.events)
4654                .with_operation_delivery(registration.manifest.operation_delivery)
4655                .with_operation_payload(registration.manifest.operation_payload)
4656                .with_flow_operations(registration.manifest.flow_operations)
4657                .with_operation_deferral(registration.manifest.operation_deferral)
4658                .with_confirm_operations(registration.manifest.confirm_operations)
4659                .with_rich_ui(registration.manifest.rich_ui)
4660                .with_role_declarations(registration.manifest.role_declarations)
4661                .with_nonce_verified(nonce_verified)
4662            })
4663            .collect();
4664        let response = ClientControlResponse::CatalogList {
4665            generation,
4666            modules: entries,
4667            subc_ops: subc_ops(),
4668        };
4669        Ok(vec![control_response_body_frame(
4670            &frame,
4671            &response,
4672            "ClientControlResponse::CatalogList",
4673        )?])
4674    }
4675
4676    /// The principal a route.open's `consumer_identity` attests: `Direct`
4677    /// when it presents none, `Reserved` when its launch nonce verifies, and
4678    /// `None` when it does not. Only a read of the supervisor's nonces, so it
4679    /// can run before any refusal check without changing which check refuses.
4680    fn attest_route_open_consumer(
4681        &self,
4682        consumer_identity: Option<&ConsumerIdentity>,
4683    ) -> Option<Principal> {
4684        let Some(consumer_identity) = consumer_identity else {
4685            return Some(Principal::Direct);
4686        };
4687        self.supervisor
4688            .spawned_consumer_authorized(
4689                &consumer_identity.module_id,
4690                &consumer_identity.launch_nonce,
4691            )
4692            .then(|| Principal::Reserved {
4693                module_id: consumer_identity.module_id.clone(),
4694            })
4695    }
4696
4697    /// The caller fields for a refusal that only has the raw request frame,
4698    /// such as the connection-level admission refusal sent before the request
4699    /// reaches [`Self::handle_route_open`].
4700    fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4701        let Ok(ClientControlRequest::RouteOpen {
4702            identity,
4703            consumer_identity,
4704            scope,
4705            ..
4706        }) = parse_client_control_request(&frame.body)
4707        else {
4708            return RouteOpenCaller::unknown();
4709        };
4710        let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4711        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4712    }
4713
4714    fn route_open_principal(
4715        &self,
4716        frame: &Frame,
4717        consumer_identity: Option<ConsumerIdentity>,
4718    ) -> Result<Result<Principal, Frame>, RouterError> {
4719        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4720        self.route_open_principal_attested(frame, consumer_identity, attested)
4721    }
4722
4723    /// [`Self::route_open_principal`] with `attested` already taken by
4724    /// [`Self::attest_route_open_consumer`] for `consumer_identity`, so one
4725    /// open checks the launch nonce once.
4726    fn route_open_principal_attested(
4727        &self,
4728        frame: &Frame,
4729        consumer_identity: Option<ConsumerIdentity>,
4730        attested: Option<Principal>,
4731    ) -> Result<Result<Principal, Frame>, RouterError> {
4732        if let Some(principal) = attested {
4733            return Ok(Ok(principal));
4734        }
4735        let Some(consumer_identity) = consumer_identity else {
4736            return Ok(Ok(Principal::Direct));
4737        };
4738
4739        Ok(Err(control_error_frame(
4740            frame,
4741            "bad_consumer_identity",
4742            format!(
4743                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4744                consumer_identity.module_id
4745            ),
4746        )?))
4747    }
4748
4749    /// Ordinary `route.open` refusals go through here; admission and breaker
4750    /// refusals log separately with their capacity or breaker state. The daemon can
4751    /// attest which code it sent: without the event, a client's "the daemon
4752    /// refused me" and the daemon's own view could only be reconciled by
4753    /// argument. Malformed input (`invalid_project_root`) does not come here;
4754    /// rejecting a request that was never a valid open is not a refusal of one.
4755    #[allow(clippy::too_many_arguments)]
4756    fn route_open_refusal_frame(
4757        &self,
4758        ctx: &RouteCtx,
4759        caller: &RouteOpenCaller,
4760        frame: &Frame,
4761        module_id: &str,
4762        reason: &'static str,
4763        code: &'static str,
4764        message: impl Into<String>,
4765    ) -> Result<Frame, RouterError> {
4766        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4767        control_error_frame(frame, code, message.into())
4768    }
4769
4770    /// Answer a waiting `route.open` whose relayed bind was refused.
4771    ///
4772    /// `daemon_authored` is provenance, not a code match: the daemon
4773    /// synthesized the refusal itself (a drain, a late reply, a scope that
4774    /// ended mid-bind) or the module refused the bind with its own body. A
4775    /// daemon refusal carries the daemon-origin bit; a module's own
4776    /// rejection is relayed with the bit clear, so a module that refuses
4777    /// with `module_reloading` still cannot mint a daemon refusal.
4778    #[allow(clippy::too_many_arguments)]
4779    fn route_bind_rejection(
4780        &self,
4781        ctx: &RouteCtx,
4782        caller: &RouteOpenCaller,
4783        frame: &Frame,
4784        target_module_id: &str,
4785        body: ErrorBody,
4786        daemon_authored: bool,
4787    ) -> Result<Vec<Frame>, RouterError> {
4788        // The daemon's own commit re-check refused the bind because the
4789        // scope ended or changed after admission. The module accepted;
4790        // counting it as a module rejection would blame the module. This
4791        // branch only runs for daemon-synthesized bodies: a module that
4792        // refuses with a scope code is answered as a module rejection below.
4793        if daemon_authored {
4794            let scope_code = match body.code.as_str() {
4795                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4796                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4797                _ => None,
4798            };
4799            if let Some(code) = scope_code {
4800                self.observe_route_open_refusal(
4801                    ctx,
4802                    caller,
4803                    target_module_id,
4804                    "scope_changed_before_commit",
4805                    code,
4806                );
4807                return Ok(vec![control_error_body_frame(frame, body)?]);
4808            }
4809        }
4810        self.counters
4811            .increment_route_open_refused("module_rejected");
4812        info!(
4813            target: "control",
4814            code = "module_rejected",
4815            module_code = ?body.code,
4816            module_id = ?target_module_id,
4817            connection_id = ctx.connection_id.get(),
4818            principal = %caller.principal,
4819            scope = %caller.scope,
4820            harness = %caller.harness,
4821            "route.open refused"
4822        );
4823        let refusal = if daemon_authored {
4824            control_error_body_frame(frame, body)?
4825        } else {
4826            control_relayed_error_body_frame(frame, body)?
4827        };
4828        Ok(vec![refusal])
4829    }
4830
4831    /// Refuse a `route.open` because the target module's bind-relay breaker is
4832    /// open, without attempting the relay.
4833    ///
4834    /// The wire code is `module_timeout`, which is the truth (the module has
4835    /// not been answering binds) and which both SDKs already classify as
4836    /// retryable with capped backoff. Reusing it is what keeps this change out
4837    /// of both SDKs; the daemon-side distinction lives in the counter key
4838    /// instead.
4839    ///
4840    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4841    /// While a breaker is open this fires on every open to that module, and the
4842    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4843    /// produced 261 lines about a single module inside 3000 lines of daemon
4844    /// log. The rare transitions are logged at warn/info instead and the volume
4845    /// is carried by the counter, so the evidence survives without the flood.
4846    /// The debug line keeps a per-refusal record reachable for whoever turns
4847    /// the level up.
4848    #[allow(clippy::too_many_arguments)]
4849    fn route_open_breaker_refusal_frame(
4850        &self,
4851        ctx: &RouteCtx,
4852        caller: &RouteOpenCaller,
4853        frame: &Frame,
4854        module_id: &str,
4855        consecutive_timeouts: u32,
4856        retry_in: Duration,
4857        probe_in_flight: bool,
4858    ) -> Result<Frame, RouterError> {
4859        self.counters
4860            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4861        debug!(
4862            target: "control",
4863            code = "module_timeout",
4864            module_id = ?module_id,
4865            connection_id = ctx.connection_id.get(),
4866            principal = %caller.principal,
4867            scope = %caller.scope,
4868            harness = %caller.harness,
4869            consecutive_timeouts,
4870            retry_in_ms = retry_in.as_millis() as u64,
4871            probe_in_flight,
4872            "route.open refused by open bind-relay breaker"
4873        );
4874        // Say what a caller can act on. An open bind-relay breaker means the
4875        // module timed out accepting several new routes in a row. The module
4876        // is still running and its established routes keep working; only new
4877        // route.open requests are refused until the cooldown ends and one
4878        // test route (the probe) gets through. A message that only counts
4879        // failed relays reads as "the module is down" to a worker that sees it.
4880        let detail = if probe_in_flight {
4881            "one test route is already being tried; retry once it settles".to_string()
4882        } else {
4883            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4884        };
4885        control_error_frame(
4886            frame,
4887            "module_timeout",
4888            format!(
4889                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4890                 timed out in a row); {detail}; its established routes are unaffected"
4891            ),
4892        )
4893    }
4894
4895    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4896    /// requester's bytes (an unknown target is whatever the client sent) and
4897    /// is Debug-formatted so control characters land in the log escaped
4898    /// rather than as terminal sequences for whoever tails it.
4899    ///
4900    /// `reason` names the check that refused, because one wire code has
4901    /// several senders: after a module registers, `target_unavailable` can
4902    /// come from a missing role, an inactive registration, a supervisor that
4903    /// has not marked the process live, a missing forwarding connection, or a
4904    /// failed relay, and a log that records only the code cannot say which of
4905    /// them fired. It is a static, daemon-chosen label per branch, so it is
4906    /// safe to print plainly and stays a closed set.
4907    fn observe_route_open_refusal(
4908        &self,
4909        ctx: &RouteCtx,
4910        caller: &RouteOpenCaller,
4911        module_id: &str,
4912        reason: &'static str,
4913        code: &'static str,
4914    ) {
4915        self.counters.increment_route_open_refused(code);
4916        info!(
4917            target: "control",
4918            code,
4919            reason,
4920            module_id = ?module_id,
4921            connection_id = ctx.connection_id.get(),
4922            principal = %caller.principal,
4923            scope = %caller.scope,
4924            harness = %caller.harness,
4925            "route.open refused"
4926        );
4927        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4928            self.route_outages.record_not_serving(module_id, reason);
4929        }
4930    }
4931
4932    /// Record an ACCEPTED route.open.
4933    ///
4934    /// Refusals have been logged and counted since the attestation work; accepts
4935    /// were invisible, so the daemon knew every principal it stamped and wrote
4936    /// none of them down. The party that attests the identity was the only party
4937    /// not recording it, which left a credential vault unable to name the sender
4938    /// of a call that reached it (claustrum #43) and left the launch-nonce
4939    /// concurrency question unanswerable from the outside.
4940    ///
4941    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4942    /// `code`/`module_id`/`connection_id` returns both directions of the same
4943    /// decision rather than two shapes a reader has to join by hand.
4944    ///
4945    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4946    /// and the difference carries information rather than being an
4947    /// inconsistency. This line is only reachable after a successful bind to a
4948    /// REGISTERED module, so the value has already passed HELLO validation
4949    /// including the path-hazard refusal and cannot contain control bytes. A
4950    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4951    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4952    ///
4953    /// Bare is also what every other daemon line already emits (`module
4954    /// registered`, `configured module supervised`). Shipping `?module_id` here
4955    /// made this instrument the only one in the file whose ids did not answer
4956    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4957    /// line whose whole purpose is being grepped beside its sibling.
4958    ///
4959    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4960    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4961    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4962    /// one are recorded identically. A test written against that harness passes
4963    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4964    /// green assertion that cannot fail. The same limit applies to the escaping
4965    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4966    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4967    /// Fencing either needs the real formatter, not the capture layer.
4968    ///
4969    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4970    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4971    /// to record. The ephemeral port would decay within minutes and answer only a
4972    /// live question. The identity question is instead answered by counting
4973    /// distinct live connections presenting one module's `consumer_identity` --
4974    /// "is anyone else holding this secret" rather than "is this the right
4975    /// process".
4976    fn observe_route_open_accept(
4977        &self,
4978        ctx: &RouteCtx,
4979        module_id: &str,
4980        principal: &str,
4981        scope: &str,
4982    ) {
4983        self.route_outages.record_accepted(module_id);
4984        self.counters.increment_route_open_accepted(principal);
4985        info!(
4986            target: "control",
4987            principal,
4988            module_id,
4989            connection_id = ctx.connection_id.get(),
4990            scope = %scope,
4991            "route.open accepted"
4992        );
4993    }
4994
4995    fn supervised_absent_route_open_refusal_frame(
4996        &self,
4997        ctx: &RouteCtx,
4998        caller: &RouteOpenCaller,
4999        frame: &Frame,
5000        module_id: &str,
5001        code: &'static str,
5002        status: &crate::supervise::ModuleStatus,
5003    ) -> Result<Frame, RouterError> {
5004        self.counters.increment_route_open_refused(code);
5005        info!(
5006            target: "control",
5007            code,
5008            reason = "supervised_not_registered",
5009            module_id = ?module_id,
5010            connection_id = ctx.connection_id.get(),
5011            principal = %caller.principal,
5012            scope = %caller.scope,
5013            harness = %caller.harness,
5014            state = %status.state,
5015            enabled = status.enabled,
5016            live = status.live,
5017            "route.open refused"
5018        );
5019        // A supervised module whose process has not registered is not
5020        // serving, whatever the reason; the supervisor knows this id, so it is
5021        // safe to track.
5022        self.route_outages
5023            .record_not_serving(module_id, "supervised_not_registered");
5024        let message = if let Some(pid) = status.waiting_for_previous_pid {
5025            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
5026                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
5027            })
5028        } else {
5029            format!(
5030                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
5031                status.state, status.enabled, status.live
5032            )
5033        };
5034        control_error_frame(frame, code, message)
5035    }
5036
5037    async fn handle_route_open(
5038        &self,
5039        ctx: &RouteCtx,
5040        frame: Frame,
5041        request: RouteOpenRequest,
5042        timings: &mut RouteOpenTimings,
5043    ) -> Result<Vec<Frame>, RouterError> {
5044        let RouteOpenRequest {
5045            target,
5046            mut identity,
5047            consumer_identity,
5048            consumer_capabilities,
5049            role_versions,
5050            busy_refusals,
5051            admission_facts,
5052            scope,
5053        } = request;
5054        let target_module_id = target_module_id(&target).to_string();
5055        timings.module_id.clone_from(&target_module_id);
5056        if self
5057            .registry
5058            .get_module_by_connection(ctx.connection_id)
5059            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5060            .is_some()
5061        {
5062            return Ok(vec![control_error_frame(
5063                &frame,
5064                "invalid_request",
5065                "module connections cannot open client routes",
5066            )?]);
5067        }
5068        debug!(
5069            connection_id = ctx.connection_id.get(),
5070            corr = frame.header.corr,
5071            module_id = %target_module_id,
5072            "handling route.open"
5073        );
5074
5075        // The caller's attested principal, requested scope and harness, for
5076        // every `route.open refused` line below. The launch nonce is checked
5077        // here, once, and the answer is reused where the principal is
5078        // enforced further down, so the order of refusals is unchanged.
5079        let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
5080        let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
5081
5082        // A malformed declaration is refused first, before anything about the
5083        // target is looked up: the same body would be refused against any
5084        // module, so the caller learns nothing by retrying or waiting. An empty
5085        // map declares nothing and travels as no field at all, so a provider
5086        // only ever sees a missing field or a non-empty one.
5087        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
5088        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
5089            self.observe_route_open_refusal(
5090                ctx,
5091                &caller,
5092                &target_module_id,
5093                "invalid_role_versions",
5094                error_codes::INVALID_REQUEST,
5095            );
5096            return Ok(vec![control_error_body_frame(
5097                &frame,
5098                ErrorBody {
5099                    code: error_codes::INVALID_REQUEST.to_string(),
5100                    message: error.to_string(),
5101                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
5102                },
5103            )?]);
5104        }
5105
5106        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
5107            if status.waiting_for_previous_pid.is_some() {
5108                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5109                    ctx,
5110                    &caller,
5111                    &frame,
5112                    &target_module_id,
5113                    "module_warming",
5114                    &status,
5115                )?]);
5116            }
5117        }
5118
5119        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
5120        // opposite. Below, a caller learns whether a module is unregistered,
5121        // supervised-but-down (with state/enabled/live), or registered without the
5122        // requested role. Elsewhere that is an enumeration leak: a probe learning
5123        // the shape of a fleet it cannot otherwise see.
5124        //
5125        // It is not one here, and the reason is the ACCESS MODEL rather than
5126        // anything about these errors. Reaching route.open requires the
5127        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
5128        // connection file, so any caller who completes it already runs as this
5129        // user -- and can read subc.jsonc for the module list and `ck module
5130        // status` for live state. The reply discloses nothing the caller cannot
5131        // read more easily from disk, while the precision is load-bearing:
5132        // `unknown_module` is retryable and a missing role is not.
5133        //
5134        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
5135        // remote transport, a sandboxed caller, a shared-host mode -- THAT
5136        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
5137        let admission_registry = self
5138            .registry
5139            .route_admission_snapshot()
5140            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5141        let Some(registration) =
5142            crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
5143        else {
5144            if let Some((status, warming)) =
5145                self.supervisor_status(&target_module_id, frame.header.corr)?
5146            {
5147                // BEFORE the two availability codes below, because for a module
5148                // that speaks no subc wire both of them are false comfort: they
5149                // say "not right now" and are retried, and this module will
5150                // never register no matter how long the caller waits. The
5151                // absence here is the declaration being honoured, not a module
5152                // that is late.
5153                if status.protocol == ModuleProtocol::None {
5154                    return Ok(vec![self.route_open_refusal_frame(
5155                        ctx,
5156                        &caller,
5157                        &frame,
5158                        &target_module_id,
5159                        "protocol_none",
5160                        error_codes::MODULE_NO_PROTOCOL,
5161                        format!(
5162                            "module_id '{target_module_id}' is declared protocol: none; \
5163                             it speaks no subc wire and serves no routes"
5164                        ),
5165                    )?]);
5166                }
5167                let code = if warming {
5168                    "module_warming"
5169                } else {
5170                    "target_unavailable"
5171                };
5172                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5173                    ctx,
5174                    &caller,
5175                    &frame,
5176                    &target_module_id,
5177                    code,
5178                    &status,
5179                )?]);
5180            }
5181            if let Some(removed_ago_ms) =
5182                self.supervisor.removal_tombstone_age_ms(&target_module_id)
5183            {
5184                return Ok(vec![self.route_open_refusal_frame(
5185                    ctx,
5186                    &caller,
5187                    &frame,
5188                    &target_module_id,
5189                    "removed",
5190                    error_codes::MODULE_REMOVED,
5191                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5192                )?]);
5193            }
5194            return Ok(vec![self.route_open_refusal_frame(
5195                ctx,
5196                &caller,
5197                &frame,
5198                &target_module_id,
5199                "not_registered",
5200                error_codes::UNKNOWN_MODULE,
5201                format!("module_id '{target_module_id}' is not registered"),
5202            )?]);
5203        };
5204
5205        // Best-effort only: registry readiness and forwarding reservation use
5206        // different locks, so a module can flip readiness between this read and
5207        // the relay. Modules must still tolerate an `on_bind` while not ready.
5208        if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
5209            .is_err()
5210        {
5211            self.counters
5212                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5213            info!(
5214                target: "control",
5215                code = error_codes::MODULE_WARMING,
5216                module_id = ?target_module_id,
5217                connection_id = ctx.connection_id.get(),
5218                principal = %caller.principal,
5219                scope = %caller.scope,
5220                harness = %caller.harness,
5221                reason = "declared_not_ready",
5222                "route.open refused"
5223            );
5224            // The module is registered but says it cannot take work, which is
5225            // an outage from the caller's side even though its process is up.
5226            self.route_outages
5227                .record_not_serving(&target_module_id, "declared_not_ready");
5228            return Ok(vec![control_error_body_frame(
5229                &frame,
5230                ErrorBody {
5231                    code: error_codes::MODULE_WARMING.to_string(),
5232                    message: format!(
5233                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5234                    ),
5235                    detail: Some(serde_json::json!({
5236                        "reason": "declared_not_ready"
5237                    })),
5238                },
5239            )?]);
5240        }
5241
5242        // Effective readiness, second half: a module that declares a capability
5243        // `need: required` is not routable while that capability has no
5244        // registered provider. It is enforced HERE, as a retryable routing
5245        // refusal, and deliberately not as spawn ordering or a boot block. The
5246        // module is still started and registered and can make its own calls;
5247        // spawn ordering is a promise that cannot be kept once a provider
5248        // crashes at runtime, and refusing to boot would stop the whole
5249        // machine, including the tools needed to fix its configuration.
5250        //
5251        // "Provided" is the evaluator's verdict, which counts a provider as
5252        // soon as it has REGISTERED, not once it is ready. Two modules that
5253        // require each other's capabilities are therefore both routable once
5254        // both register; counting readiness instead would deadlock them.
5255        //
5256        // Only new opens are refused. Routes already bound when a provider
5257        // goes away stay bound: nothing here tears them down, and the module
5258        // answers them as it can. Like the readiness read above this is
5259        // best-effort against a provider registering or leaving concurrently.
5260        if let Some(capability) = self.first_unprovided_required_capability(registration) {
5261            self.counters
5262                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5263            info!(
5264                target: "control",
5265                code = error_codes::MODULE_WARMING,
5266                module_id = ?target_module_id,
5267                connection_id = ctx.connection_id.get(),
5268                principal = %caller.principal,
5269                scope = %caller.scope,
5270                harness = %caller.harness,
5271                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5272                capability = %capability,
5273                "route.open refused"
5274            );
5275            return Ok(vec![control_error_body_frame(
5276                &frame,
5277                ErrorBody {
5278                    code: error_codes::MODULE_WARMING.to_string(),
5279                    message: format!(
5280                        "module_id '{target_module_id}' requires capability '{capability}', \
5281                         which no registered module provides; retry"
5282                    ),
5283                    detail: Some(serde_json::json!({
5284                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5285                        "capability": capability,
5286                    })),
5287                },
5288            )?]);
5289        }
5290
5291        if !target_has_required_role(&target, &registration.manifest.provides) {
5292            return Ok(vec![self.route_open_refusal_frame(
5293                ctx,
5294                &caller,
5295                &frame,
5296                &target_module_id,
5297                "role_not_provided",
5298                "target_unavailable",
5299                format!("module_id '{target_module_id}' does not provide the requested target"),
5300            )?]);
5301        }
5302
5303        if registration.state != ChannelState::Active {
5304            return Ok(vec![self.route_open_refusal_frame(
5305                ctx,
5306                &caller,
5307                &frame,
5308                &target_module_id,
5309                "registration_not_active",
5310                "target_unavailable",
5311                format!("module_id '{target_module_id}' is not active"),
5312            )?]);
5313        }
5314
5315        if self
5316            .forwarding
5317            .module_is_draining(&target_module_id)
5318            .map_err(RouterError::Forwarding)?
5319        {
5320            return Ok(vec![self.route_open_refusal_frame(
5321                ctx,
5322                &caller,
5323                &frame,
5324                &target_module_id,
5325                "reloading",
5326                "module_reloading",
5327                format!("module_id '{target_module_id}' is reloading"),
5328            )?]);
5329        }
5330
5331        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5332            process_liveness.process_live(&target_module_id) == Some(false)
5333        }) {
5334            // A module the supervisor is restarting or reloading can still hold
5335            // a registration: the old process before its connection closes, or
5336            // a new one that registered while the supervisor was draining. The
5337            // forwarding table does not see that as draining, but the consumer
5338            // should still be told to retry soon, exactly as for the drain
5339            // above, rather than that the target is unavailable.
5340            if process_liveness.process_replacing(&target_module_id) {
5341                return Ok(vec![self.route_open_refusal_frame(
5342                    ctx,
5343                    &caller,
5344                    &frame,
5345                    &target_module_id,
5346                    "reloading",
5347                    "module_reloading",
5348                    format!("module_id '{target_module_id}' is reloading"),
5349                )?]);
5350            }
5351            return Ok(vec![self.route_open_refusal_frame(
5352                ctx,
5353                &caller,
5354                &frame,
5355                &target_module_id,
5356                "supervisor_not_live",
5357                "target_unavailable",
5358                format!("module_id '{target_module_id}' is not live"),
5359            )?]);
5360        }
5361
5362        if !self
5363            .forwarding
5364            .has_live_module_connection(&target_module_id)
5365            .map_err(RouterError::Forwarding)?
5366        {
5367            return Ok(vec![self.route_open_refusal_frame(
5368                ctx,
5369                &caller,
5370                &frame,
5371                &target_module_id,
5372                "no_forwarding_connection",
5373                "target_unavailable",
5374                format!("module_id '{target_module_id}' has no live forwarding connection"),
5375            )?]);
5376        }
5377
5378        if let Some(error) =
5379            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5380        {
5381            self.observe_route_open_refusal(
5382                ctx,
5383                &caller,
5384                &target_module_id,
5385                "op_not_allowed",
5386                "op_not_allowed",
5387            );
5388            return Ok(vec![error]);
5389        }
5390
5391        let principal =
5392            match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5393                Ok(principal) => principal,
5394                Err(error) => {
5395                    self.observe_route_open_refusal(
5396                        ctx,
5397                        &caller,
5398                        &target_module_id,
5399                        "bad_consumer_identity",
5400                        "bad_consumer_identity",
5401                    );
5402                    return Ok(vec![error]);
5403                }
5404            };
5405        timings.principal = principal_label(&principal);
5406
5407        // Only Callosum can attest a production phone build. Consume the claim
5408        // here so it can be kept on the route without reaching any module.
5409        let build_class = identity.build_class.take().filter(|build_class| {
5410            if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5411                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5412            } else {
5413                if self
5414                    .build_class_warned
5415                    .lock()
5416                    .unwrap_or_else(|p| p.into_inner())
5417                    .insert(ctx.connection_id)
5418                {
5419                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5420                        "ignoring build_class from a non-Callosum opener");
5421                }
5422                false
5423            }
5424        });
5425
5426        // This is attested, control-plane policy for supervised module origins.
5427        // Keep it before route reservation and out of the opaque forwarding hot
5428        // path: data frames must never acquire a per-frame capability check.
5429        if let Principal::Reserved {
5430            module_id: opening_module_id,
5431        } = &principal
5432        {
5433            if let Some(opening_registration) = admission_registry
5434                .iter()
5435                .find(|r| r.manifest.module_id == *opening_module_id)
5436            {
5437                if let Some(capability) =
5438                    denied_capability(&opening_registration.manifest, &registration.manifest)
5439                {
5440                    warn!(
5441                        opening_module_id,
5442                        target_module_id,
5443                        capability,
5444                        "refusing route.open because an attested capability deny edge matches"
5445                    );
5446                    return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5447                        &frame,
5448                        &target_module_id,
5449                        "capability_deny_edge",
5450                        "capability_forbidden",
5451                        format!(
5452                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5453                        ),
5454                    )?]);
5455                }
5456            }
5457        }
5458
5459        if admission_facts.is_some() {
5460            let carrier_matches = matches!(
5461                &principal,
5462                Principal::Reserved { module_id }
5463                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5464            );
5465            if !carrier_matches {
5466                return Ok(vec![self.route_open_refusal_frame(
5467                    ctx,
5468                    &caller,
5469                    &frame,
5470                    &target_module_id,
5471                    "admission_facts_carrier_not_permitted",
5472                    "admission_facts_not_permitted",
5473                    "admission facts may only be carried by the configured reserved module",
5474                )?]);
5475            }
5476
5477            let target_allowed = self
5478                .admission_facts_targets
5479                .as_ref()
5480                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5481            if !target_allowed {
5482                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5483                    &frame,
5484                    &target_module_id,
5485                    "admission_facts_target_not_listed",
5486                    "admission_facts_target_not_allowed",
5487                    format!(
5488                        "admission facts are not permitted for target module_id '{target_module_id}'"
5489                    ),
5490                )?]);
5491            }
5492
5493            // Keep the value opaque to subc. The downstream admission validator owns
5494            // schema and semantic checks; this daemon only enforces carrier authority
5495            // and the configured destination allowlist.
5496        }
5497
5498        // Scope admission, on the attested principal above and never on the
5499        // request body. The tag read here travels with the pending bind and is
5500        // compared with the published one at commit, so a sync between here
5501        // and the module's ack refuses the open instead of binding a stamp
5502        // that is no longer true.
5503        let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
5504            None => (None, None, false),
5505            Some(selector) => {
5506                let owner_configured = match &selector.owner {
5507                    // A reserved owner counts as configured from before its
5508                    // process is spawned (see `SupervisorHandle::is_configured`).
5509                    // So an owner that has not synced its scopes yet is refused
5510                    // as retryable (`scope_not_synced`), not as one that will
5511                    // never sync.
5512                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5513                    _ => false,
5514                };
5515                let mut scopes = self
5516                    .scopes
5517                    .write()
5518                    .unwrap_or_else(|poisoned| poisoned.into_inner());
5519                let admitted = crate::scopes::admission::admit_route(
5520                    &scopes,
5521                    &admission_registry,
5522                    &principal,
5523                    &target_module_id,
5524                    Some(selector),
5525                    owner_configured,
5526                    crate::scopes::admission::AdmissionMode::Open,
5527                )
5528                .map(|route| {
5529                    let admission = route.scope.expect("scoped admission");
5530                    let bound = scopes.reserve_bind(
5531                        &admission,
5532                        &principal,
5533                        ctx.connection_id,
5534                        selector.hold_presence,
5535                    );
5536                    (admission, bound)
5537                });
5538                match admitted {
5539                    Ok((admission, bound)) => {
5540                        (Some(bound), Some(admission.stamp), admission.stamp_required)
5541                    }
5542                    Err(refusal) => {
5543                        return Ok(vec![self.route_open_refusal_frame(
5544                            ctx,
5545                            &caller,
5546                            &frame,
5547                            &target_module_id,
5548                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5549                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5550                            refusal.message,
5551                        )?]);
5552                    }
5553                }
5554            }
5555        };
5556        // From here on a refusal names the scope by the stamp admission
5557        // issued, exactly as the `route.open accepted` line does.
5558        if let Some(stamp) = &scope_stamp {
5559            caller.scope = route_open_scope_label(stamp);
5560        }
5561
5562        // Allow resolvable missing-tail paths so a vanished directory does not
5563        // prevent reopening an existing session for cleanup. Binding does not
5564        // authorize creation of durable provider state.
5565        //
5566        // Resolve real paths rather than lexical cleanup: canonicalize the
5567        // longest existing ancestor and append the missing tail, retaining the
5568        // path's real existing ancestry even when a leaf is missing.
5569        timings.enter(RouteOpenPhase::ProjectRoot);
5570        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5571            Ok(project) => project,
5572            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5573        };
5574        timings.enter(RouteOpenPhase::Admission);
5575        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5576            Ok(root) => root,
5577            Err(err) => {
5578                return Ok(vec![control_error_frame(
5579                    &frame,
5580                    "invalid_project_root",
5581                    err.to_string(),
5582                )?])
5583            }
5584        };
5585        // Compute compatibility project fields before stripping optional stamp
5586        // members. Preserve the canonical opener root and its project ID,
5587        // including absence. For rootless openers, use stamp workdir or `/`, and
5588        // opener ID, else stamp ID, else no ID. Emit these identity fields only
5589        // to targets without bind-no-project/v1.
5590        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5591
5592        // Last gate before any relay work, and deliberately after the cheap
5593        // registry and availability checks above: those name a more precise
5594        // condition (unknown, removed, reloading) and a caller is better served
5595        // by the precise code than by this one.
5596        //
5597        // Everything below this point costs an egress permit, a reserved handle
5598        // pair and, if the module does not answer, the whole relay budget. The
5599        // reader no longer waits for that budget, so cap each target explicitly;
5600        // serial dispatch used to provide the accidental cap of one relay per
5601        // connection. Admission is a mutex-protected count and never waits.
5602        let _concurrency_guard = match self
5603            .route_bind_concurrency
5604            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5605        {
5606            Ok(guard) => guard,
5607            Err(in_flight) => {
5608                return Ok(vec![self.route_open_target_capacity_refusal(
5609                    ctx,
5610                    &caller,
5611                    &frame,
5612                    &target_module_id,
5613                    in_flight,
5614                )?]);
5615            }
5616        };
5617
5618        // A module that has already burned the whole budget `threshold` times
5619        // in a row does not get to charge it again until a probe says it recovered.
5620        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5621            RouteBindAdmission::Admitted { guard, probe } => {
5622                if probe {
5623                    info!(
5624                        module_id = %target_module_id,
5625                        connection_id = ctx.connection_id.get(),
5626                        "route.bind breaker half-open: admitting one probe"
5627                    );
5628                }
5629                guard
5630            }
5631            RouteBindAdmission::Refused {
5632                consecutive_timeouts,
5633                retry_in,
5634                probe_in_flight,
5635            } => {
5636                return Ok(vec![self.route_open_breaker_refusal_frame(
5637                    ctx,
5638                    &caller,
5639                    &frame,
5640                    &target_module_id,
5641                    consecutive_timeouts,
5642                    retry_in,
5643                    probe_in_flight,
5644                )?]);
5645            }
5646        };
5647
5648        // Resolve the per-module budget here so the wait matches the operator's
5649        // intent for this specific target. A per-module override in
5650        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
5651        // daemons) wins over the daemon-wide default.
5652        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5653        let relay_deadline = Instant::now() + route_bind_relay_timeout;
5654        let operator_app = self
5655            .operator_apps
5656            .lock()
5657            .unwrap_or_else(|p| p.into_inner())
5658            .get(&ctx.connection_id)
5659            .cloned();
5660        timings.enter(RouteOpenPhase::EgressReservation);
5661        let pending = match self
5662            .forwarding
5663            .begin_route_bind_relay_for(
5664                ctx.connection_id,
5665                ctx.egress.clone(),
5666                response_version(&frame),
5667                frame.header.corr,
5668                &target_module_id,
5669                principal.clone(),
5670                bound_scope,
5671                project_root,
5672                identity.harness.clone(),
5673                build_class,
5674                operator_app.clone(),
5675                relay_deadline,
5676                busy_refusals == Some(true),
5677            )
5678            .await
5679        {
5680            Ok(pending) => pending,
5681            Err(err) => {
5682                return Ok(vec![self.route_open_refusal_frame(
5683                    ctx,
5684                    &caller,
5685                    &frame,
5686                    &target_module_id,
5687                    "relay_reservation_failed",
5688                    forwarding_error_code(&err),
5689                    err.to_string(),
5690                )?])
5691            }
5692        };
5693        timings.enter(RouteOpenPhase::Admission);
5694        let crate::forwarding::PendingRouteBindRelay {
5695            endpoint,
5696            module_sink,
5697            negotiated_ver,
5698            client_channel,
5699            client_epoch,
5700            module_channel,
5701            module_epoch,
5702            corr: relay_corr,
5703            receiver,
5704        } = pending;
5705        let mut reservation =
5706            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5707
5708        // Reserving egress can wait while a module reconnects or a swap cuts
5709        // over. Check the relay's captured connection, not the earlier by-id
5710        // lookup: the original module's flow/run capabilities cannot authorize
5711        // its replacement. The captured sink stays bound to that connection.
5712        let relay_registration = self
5713            .registry
5714            .get_module_by_connection(endpoint.connection_id)
5715            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5716        let relay_capabilities = relay_registration
5717            .as_ref()
5718            .and_then(|registration| registration.manifest.capabilities.as_ref());
5719        if let Some(selector) = &scope {
5720            // Egress reservation may wait across a scope update. Recheck the
5721            // proof against the current record before sending the captured bind.
5722            if let Err(refusal) = self
5723                .scopes
5724                .read()
5725                .unwrap_or_else(|poisoned| poisoned.into_inner())
5726                .check_bind_proof(&principal, selector)
5727            {
5728                reservation.release_and_disarm();
5729                return Ok(vec![self.route_open_refusal_frame(
5730                    ctx,
5731                    &caller,
5732                    &frame,
5733                    &target_module_id,
5734                    refusal.code,
5735                    refusal.code,
5736                    refusal.message,
5737                )?]);
5738            }
5739        }
5740        if let Some(stamp) = scope_stamp.as_ref() {
5741            if let Err(refusal) = crate::scopes::check_target_required_stamp(
5742                stamp_required,
5743                &target_module_id,
5744                relay_capabilities,
5745            )
5746            .and_then(|()| {
5747                crate::scopes::check_target_flow_support(
5748                    stamp,
5749                    &target_module_id,
5750                    relay_capabilities,
5751                )
5752            })
5753            .and_then(|()| {
5754                crate::scopes::check_target_agent_run_support(
5755                    stamp,
5756                    &target_module_id,
5757                    relay_capabilities,
5758                )
5759            }) {
5760                reservation.release_and_disarm();
5761                return Ok(vec![self.route_open_refusal_frame(
5762                    ctx,
5763                    &caller,
5764                    &frame,
5765                    &target_module_id,
5766                    refusal.code,
5767                    refusal.code,
5768                    refusal.message,
5769                )?]);
5770            }
5771        }
5772
5773        // Older readers may reject unknown fields. Select optional fields from
5774        // the captured endpoint's declaration, including on unscoped binds.
5775        if scope_stamp
5776            .as_ref()
5777            .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5778            && !crate::scopes::target_supports_federated(relay_capabilities)
5779        {
5780            // Keep bound_scope: even an unstamped legacy route belongs to this epoch.
5781            scope_stamp = None;
5782        }
5783        if !relay_capabilities.is_some_and(|caps| {
5784            caps.provides
5785                .iter()
5786                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5787        }) {
5788            if let Some(stamp) = scope_stamp.as_mut() {
5789                stamp.workdir = None;
5790                stamp.project_id = None;
5791            }
5792        }
5793        // A worker run has no agent_id; on_behalf_of_agent is its only agent
5794        // attribution. The run gate above has already required the target to
5795        // declare `worker-runs/v1`, which promises to read both
5796        // `on_behalf_of_agent` and `call_mode`.
5797        let worker_run = scope_stamp
5798            .as_ref()
5799            .is_some_and(crate::scopes::is_worker_run)
5800            && relay_capabilities.is_some_and(|caps| {
5801                caps.provides
5802                    .iter()
5803                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5804            });
5805        if !worker_run
5806            && !relay_capabilities.is_some_and(|caps| {
5807                caps.provides
5808                    .iter()
5809                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5810            })
5811        {
5812            if let Some(stamp) = scope_stamp.as_mut() {
5813                stamp.attributes.on_behalf_of_agent = None;
5814            }
5815        }
5816        if !relay_capabilities.is_some_and(|caps| {
5817            caps.provides
5818                .iter()
5819                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5820        }) {
5821            if let Some(stamp) = scope_stamp.as_mut() {
5822                stamp.attributes.flow_version = None;
5823                stamp.attributes.flow_code_hash = None;
5824                stamp.attributes.flow_install_id = None;
5825            }
5826        }
5827        let call_mode = if worker_run
5828            || relay_capabilities
5829                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5830        {
5831            Some(derive_call_mode(scope_stamp.as_ref()))
5832        } else {
5833            if let Some(stamp) = scope_stamp.as_mut() {
5834                stamp.expires_at_ms = None;
5835            }
5836            None
5837        };
5838
5839        debug!(
5840            connection_id = ctx.connection_id.get(),
5841            client_channel,
5842            client_epoch,
5843            module_channel,
5844            module_epoch,
5845            "reserved route handle pair"
5846        );
5847        // Rendered BEFORE the move into the relay, because the accept arm below
5848        // is where it is logged and the principal is gone by then.
5849        let route_principal_label = principal_label(&principal);
5850        let relay = ModuleControlRequest::RouteBind {
5851            route_channel: module_channel,
5852            epoch: module_epoch,
5853            target,
5854            identity,
5855            principal: Some(principal),
5856            consumer_capabilities,
5857            role_versions,
5858            admission_facts,
5859            scope: scope_stamp,
5860            call_mode,
5861        };
5862        let peer = RouteBindPeer::new()
5863            .with_observed_peer(
5864                relay_capabilities
5865                    .is_some_and(|caps| {
5866                        caps.provides
5867                            .iter()
5868                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5869                    })
5870                    .then(|| {
5871                        self.observed_peers
5872                            .lock()
5873                            .unwrap_or_else(|p| p.into_inner())
5874                            .get(&ctx.connection_id)
5875                            .cloned()
5876                            .unwrap_or_else(ObservedPeer::unobserved)
5877                    }),
5878            )
5879            .with_operator_app(
5880                self.connection_operator_app(operator_app.as_ref())
5881                    .filter(|_| {
5882                        relay_capabilities.is_some_and(|caps| {
5883                            caps.provides
5884                                .iter()
5885                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5886                        })
5887                    }),
5888            );
5889        let relay_body = crate::bind_compat::encode_bind(
5890            &relay,
5891            &peer,
5892            &legacy_bind_project,
5893            relay_capabilities,
5894        )
5895        .map_err(|err| {
5896            RouterError::backend(
5897                0,
5898                frame.header.corr,
5899                format!("failed to encode route.bind request: {err}"),
5900            )
5901        })?;
5902        let relay_frame = Frame::build_with_version(
5903            negotiated_ver,
5904            FrameType::Request,
5905            control_flags(),
5906            0,
5907            0,
5908            relay_corr,
5909            relay_body,
5910        )
5911        .map_err(RouterError::FrameBuild)?;
5912
5913        timings.enter(RouteOpenPhase::BindSend);
5914        let sent = module_sink.send(relay_frame).await;
5915        timings.enter(RouteOpenPhase::Admission);
5916        if let Err(err) = sent {
5917            reservation.release_and_disarm();
5918            return Ok(vec![self.route_open_refusal_frame(
5919                ctx,
5920                &caller,
5921                &frame,
5922                &target_module_id,
5923                "relay_send_failed",
5924                "target_unavailable",
5925                err.to_string(),
5926            )?]);
5927        }
5928
5929        #[cfg(test)]
5930        if let Some(hold) = &self.route_bind_relay_sent_hold {
5931            // First wait: tell the test the bind is queued. Second wait: stay
5932            // here until the test has driven whatever it wants to happen
5933            // between queueing the bind and marking it enqueued.
5934            hold.wait().await;
5935            hold.wait().await;
5936        }
5937
5938        // A missing relay is not by itself an abandoned bind: the module may
5939        // already have acked it and the route may be committed. Only a relay
5940        // dropped without any module reply needs this GOODBYE; one a reply
5941        // settled has already had its GOODBYE decided by the reply path.
5942        if self
5943            .forwarding
5944            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5945            .map_err(RouterError::Forwarding)?
5946            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5947        {
5948            self.send_abandoned_route_bind_goodbye(
5949                &module_sink,
5950                negotiated_ver,
5951                module_channel,
5952                module_epoch,
5953            );
5954        }
5955
5956        timings.enter(RouteOpenPhase::BindAckWait);
5957        let outcome = timeout_at(relay_deadline, receiver).await;
5958        timings.enter(RouteOpenPhase::Admission);
5959        match outcome {
5960            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5961                reservation.disarm();
5962                if breaker.record_accepted() {
5963                    info!(
5964                        module_id = %target_module_id,
5965                        "route.bind breaker closed: the probe was accepted"
5966                    );
5967                }
5968                self.observe_route_open_accept(
5969                    ctx,
5970                    &target_module_id,
5971                    &route_principal_label,
5972                    &caller.scope,
5973                );
5974                Ok(Vec::new())
5975            }
5976            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5977                reservation.release_and_disarm();
5978                // A module that says no in microseconds is healthy. Rejection
5979                // is a different condition with its own refusal and must not
5980                // move the breaker.
5981                breaker.record_inconclusive();
5982                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5983            }
5984            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5985                reservation.release_and_disarm();
5986                breaker.record_inconclusive();
5987                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5988            }
5989            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5990                reservation.release_and_disarm();
5991                breaker.record_inconclusive();
5992                // Fires when the module's connection closes while a relayed
5993                // bind is pending -- typically a caller racing a module restart
5994                // whose bind was relayed BEFORE the drain mark went up. Logged
5995                // because the caller sees only its own error and the fleet has
5996                // already spent one diagnosis round unable to tell this arm
5997                // from a relay timeout without daemon-side evidence.
5998                tracing::warn!(
5999                    module_id = %target_module_id,
6000                    "route.bind relay abandoned: {message}"
6001                );
6002                Ok(vec![self.route_open_refusal_frame(
6003                    ctx,
6004                    &caller,
6005                    &frame,
6006                    &target_module_id,
6007                    "relay_abandoned",
6008                    "target_unavailable",
6009                    message,
6010                )?])
6011            }
6012            Ok(Err(_)) => {
6013                reservation.release_and_disarm();
6014                breaker.record_inconclusive();
6015                Ok(vec![self.route_open_refusal_frame(
6016                    ctx,
6017                    &caller,
6018                    &frame,
6019                    &target_module_id,
6020                    "relay_waiter_canceled",
6021                    "target_unavailable",
6022                    "route.bind relay waiter was canceled before the module responded",
6023                )?])
6024            }
6025            Err(_) => {
6026                reservation.release_and_disarm();
6027                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
6028                // answer at all is the one condition a fast refusal can
6029                // usefully stand in for; every other arm already answered.
6030                if let Some(opened) = breaker.record_timeout(
6031                    self.route_bind_breaker_threshold,
6032                    self.route_bind_breaker_cooldown,
6033                ) {
6034                    warn!(
6035                        module_id = %target_module_id,
6036                        consecutive_timeouts = opened.consecutive_timeouts,
6037                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
6038                        reopened_after_probe = opened.reopened_after_probe,
6039                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
6040                    );
6041                }
6042                // The generous budget just burned to no answer: the module is
6043                // registered and its connection is up, but its bind handler sat
6044                // on the ack for the full budget (warm-on-bind, cold configure,
6045                // or a wedged handler). Every earlier unavailability shape
6046                // fast-refuses BEFORE the relay, so this arm firing means the
6047                // slowness is module-side -- log it so the per-module timeline
6048                // is reconstructable without client audit rows.
6049                tracing::warn!(
6050                    module_id = %target_module_id,
6051                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
6052                    "route.bind relay timed out: module did not ack within budget"
6053                );
6054                Ok(vec![self.route_open_refusal_frame(
6055                    ctx,
6056                    &caller,
6057                    &frame,
6058                    &target_module_id,
6059                    "relay_timed_out",
6060                    "module_timeout",
6061                    format!(
6062                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
6063                        route_bind_relay_timeout
6064                    ),
6065                )?])
6066            }
6067        }
6068    }
6069
6070    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6071        let response = ClientControlResponse::SupervisorSpawnSnapshot {
6072            snapshot: self.supervisor.spawn_snapshot(),
6073        };
6074        Ok(vec![control_response_body_frame(
6075            &frame,
6076            &response,
6077            "ClientControlResponse::SupervisorSpawnSnapshot",
6078        )?])
6079    }
6080
6081    fn handle_supervisor_spawn_subscribe(
6082        &self,
6083        ctx: &RouteCtx,
6084        frame: Frame,
6085        since: Option<SpawnCursor>,
6086    ) -> Result<Vec<Frame>, RouterError> {
6087        match self.supervisor.subscribe_spawns(
6088            ctx.connection_id,
6089            frame.header.corr,
6090            response_version(&frame),
6091            since,
6092            ctx.egress.clone(),
6093        ) {
6094            Ok(()) => Ok(Vec::new()),
6095            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
6096                Ok(vec![control_error_body_frame(
6097                    &frame,
6098                    ErrorBody {
6099                        code: "spawn_cursor_incarnation_mismatch".to_string(),
6100                        message: "spawn cursor belongs to a different daemon incarnation"
6101                            .to_string(),
6102                        detail: Some(serde_json::json!({
6103                            "current_daemon_incarnation": current
6104                        })),
6105                    },
6106                )?])
6107            }
6108            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
6109                &frame,
6110                ErrorBody {
6111                    code: "spawn_cursor_too_old".to_string(),
6112                    message: "spawn cursor predates the retained event ring".to_string(),
6113                    detail: Some(serde_json::json!({
6114                        "oldest_retained_cursor": oldest
6115                    })),
6116                },
6117            )?]),
6118            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
6119                0,
6120                frame.header.corr,
6121                format!("failed to open supervisor spawn subscription: {error}"),
6122            )),
6123        }
6124    }
6125
6126    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6127        // Status/configuration snapshots may wait on synchronous supervisor
6128        // locks. Keep that work off the connection's async runtime thread.
6129        let handler = self.clone();
6130        let runtime = tokio::runtime::Handle::current();
6131        // The blocking pool does not inherit the control request's tracing
6132        // subscriber. Preserve it so slow snapshot-lock warnings stay visible.
6133        let subscriber = tracing::dispatcher::get_default(Clone::clone);
6134        let corr = frame.header.corr;
6135        tokio::task::spawn_blocking(move || {
6136            runtime.block_on(
6137                list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6138                    .with_subscriber(subscriber),
6139            )
6140        })
6141        .await
6142        .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6143    }
6144
6145    async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6146        let generation = self
6147            .registry
6148            .generation()
6149            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6150        let supervised = self.supervisor.list();
6151        let mut statuses = BTreeMap::new();
6152        let mut not_running = BTreeSet::new();
6153        for module in &supervised {
6154            let status = module.status_for_control("list").map_err(|err| {
6155                RouterError::backend(
6156                    0,
6157                    frame.header.corr,
6158                    format!("failed to read supervisor status: {err}"),
6159                )
6160            })?;
6161            if !status.enabled || !status.process_alive {
6162                not_running.insert(module.module_id().to_owned());
6163            }
6164            statuses.insert(module.module_id().to_owned(), status);
6165        }
6166        // Legacy per-process diagnostics keep their shared optional budget.
6167        // The cached tree and accounting export below does not spend it.
6168        let mut process_resources = BTreeMap::new();
6169        for module in &supervised {
6170            process_resources.insert(
6171                module.module_id().to_owned(),
6172                crate::subtree_resources::read_process(module.clone()).await,
6173            );
6174        }
6175        // Image evidence uses the same optional deadline, but an exhausted
6176        // budget must not suppress the committed resource view.
6177        let mut image_readings = BTreeMap::new();
6178        for module in &supervised {
6179            image_readings.insert(
6180                module.module_id().to_owned(),
6181                module.running_image_agreement().await,
6182            );
6183        }
6184        let readings = self.resource_tracker.list(supervised.clone());
6185        let mut modules = Vec::new();
6186        for module in supervised {
6187            let status = statuses
6188                .remove(module.module_id())
6189                .expect("each supervised module has a status snapshot");
6190            let (configured, _) = module.configuration().map_err(|err| {
6191                RouterError::backend(
6192                    0,
6193                    frame.header.corr,
6194                    format!("failed to read module configuration: {err}"),
6195                )
6196            })?;
6197            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6198                RouterError::backend(
6199                    0,
6200                    frame.header.corr,
6201                    format!("failed to read retry time: {err}"),
6202                )
6203            })?;
6204            let image = image_readings
6205                .remove(module.module_id())
6206                .expect("each supervised module has an image snapshot");
6207            let resources = process_resources.remove(module.module_id());
6208            let mut verdict =
6209                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6210            verdict.spec = Some(launch_spec_agreement(
6211                &configured,
6212                status.process_alive,
6213                status.launch_spec_digest.as_ref(),
6214            ));
6215            let pending_reload = Some(verdict);
6216            let configured_enabled = self
6217                .capability_evaluator
6218                .configured_enabled(&status.module_id);
6219            // Keep the retired policy field on the wire for one release so
6220            // existing status consumers still receive the platform policy.
6221            modules.push(
6222                SupervisorEntry::new(
6223                    status.module_id,
6224                    status.state.to_string(),
6225                    status.enabled,
6226                    status.live,
6227                    status.health.status,
6228                )
6229                .with_configured_enabled(configured_enabled)
6230                .with_next_retry_at_ms(next_retry_at_ms)
6231                .with_launch_nonce_env(Some(!cfg!(unix)))
6232                .with_launch_nonce_source(status.launch_nonce_source)
6233                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6234                .with_protocol(status.protocol)
6235                .with_pending_reload(pending_reload)
6236                .with_last_probe_ms(status.health.last_probe_ms)
6237                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6238                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6239                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6240                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6241                .with_restart_count(Some(status.restart_count))
6242                .with_max_restarts(Some(status.max_restarts))
6243                .with_lifetime_restarts(Some(status.lifetime_restarts))
6244                .with_spawn_generation(Some(status.spawn_generation))
6245                .with_restart_window_secs(Some(status.restart_window.as_secs()))
6246                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6247                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6248                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6249                .with_resources(resources),
6250            );
6251        }
6252        for module in &mut modules {
6253            if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6254                module.tree = tree.clone();
6255                module.cpu_last_hour = hour.clone();
6256            }
6257            module.tree_unavailable = readings
6258                .reasons
6259                .get(&module.module_id)
6260                .map(|reason| (*reason).to_owned());
6261            if module.tree.is_none() {
6262                module.tree_unavailable = Some(
6263                    if not_running.contains(&module.module_id) {
6264                        "not_running"
6265                    } else if let Some(reason) = readings.reasons.get(&module.module_id) {
6266                        reason
6267                    } else {
6268                        "not_yet_measured"
6269                    }
6270                    .to_owned(),
6271                );
6272            }
6273        }
6274        let response = ClientControlResponse::SupervisorList {
6275            generation,
6276            modules,
6277            daemon: readings.daemon,
6278            agent_commands: readings.agent_commands,
6279        };
6280        Ok(vec![control_response_body_frame(
6281            &frame,
6282            &response,
6283            "ClientControlResponse::SupervisorList",
6284        )?])
6285    }
6286
6287    fn handle_supervisor_stderr_tail(
6288        &self,
6289        frame: Frame,
6290        module_id: String,
6291        max_lines: Option<u32>,
6292        max_bytes: Option<u32>,
6293    ) -> Result<Vec<Frame>, RouterError> {
6294        let Some(module) = self.supervisor.get(&module_id) else {
6295            return Ok(vec![control_error_frame(
6296                &frame,
6297                "unknown_module",
6298                format!("module_id '{module_id}' is not supervised"),
6299            )?]);
6300        };
6301
6302        let snapshot = module.stderr_tail(
6303            max_lines.map(|value| value as usize),
6304            max_bytes.map(|value| value as usize),
6305        );
6306
6307        let response = ClientControlResponse::SupervisorStderrTail {
6308            module_id,
6309            tail: StderrTail {
6310                capture: match snapshot.capture {
6311                    CaptureState::Captured => StderrCaptureState::Captured,
6312                    CaptureState::Incomplete { reason } => {
6313                        StderrCaptureState::Incomplete { reason }
6314                    }
6315                    CaptureState::NotCaptured { reason } => {
6316                        StderrCaptureState::NotCaptured { reason }
6317                    }
6318                },
6319                entries: snapshot
6320                    .entries
6321                    .into_iter()
6322                    .map(|entry| match entry {
6323                        TailEntry::Line {
6324                            text,
6325                            truncated,
6326                            at_ms,
6327                        } => StderrTailEntry::Line {
6328                            text,
6329                            truncated,
6330                            at_ms,
6331                        },
6332                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6333                    })
6334                    .collect(),
6335                dropped_lines: snapshot.dropped_lines,
6336            },
6337        };
6338        Ok(vec![control_response_body_frame(
6339            &frame,
6340            &response,
6341            "ClientControlResponse::SupervisorStderrTail",
6342        )?])
6343    }
6344
6345    async fn observe_control_wait<F: std::future::Future>(
6346        &self,
6347        ctx: &RouteCtx,
6348        target: Option<&str>,
6349        kind: &'static str,
6350        future: F,
6351    ) -> F::Output {
6352        self.counters
6353            .observe_reader_wait(
6354                || {
6355                    let module = self
6356                        .forwarding
6357                        .module_id_for_connection(ctx.connection_id)
6358                        .ok()
6359                        .flatten();
6360                    let principal = if module.is_some() { "module" } else { "direct" };
6361                    (
6362                        self.resolve_reader_wait_target(kind, target)
6363                            .or(module)
6364                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6365                        principal.into(),
6366                    )
6367                },
6368                kind,
6369                future,
6370            )
6371            .await
6372    }
6373
6374    /// Resolve the reader-wait module ID only from supervisor or registry
6375    /// records; never attribute a wait from the request selector alone.
6376    pub(crate) fn resolve_reader_wait_target(
6377        &self,
6378        op: &str,
6379        selector: Option<&str>,
6380    ) -> Option<String> {
6381        let selector = selector?;
6382        match op {
6383            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6384                .supervisor
6385                .get(selector)
6386                .map(|module| module.module_id().to_owned()),
6387            ops::SUPERVISOR_HEALTH_PROBE => self
6388                .registry
6389                .get_module(selector)
6390                .ok()
6391                .flatten()
6392                .map(|registration| registration.manifest.module_id),
6393            _ => None,
6394        }
6395    }
6396
6397    async fn handle_supervisor_terminals(
6398        &self,
6399        ctx: &RouteCtx,
6400        frame: Frame,
6401        module_id: String,
6402    ) -> Result<Vec<Frame>, RouterError> {
6403        let Some(module) = self.supervisor.get(&module_id) else {
6404            return Ok(vec![control_error_frame(
6405                &frame,
6406                "unknown_module",
6407                format!("module_id '{module_id}' is not supervised"),
6408            )?]);
6409        };
6410
6411        // The journal read runs on a blocking thread: it can be megabytes of
6412        // file I/O and must not occupy a runtime worker.
6413        let terminals = self
6414            .observe_control_wait(
6415                ctx,
6416                Some(module.module_id()),
6417                ops::SUPERVISOR_TERMINALS,
6418                module.read_durable_terminal_history(),
6419            )
6420            .await
6421            .map_err(|error| {
6422                RouterError::backend(
6423                    0,
6424                    frame.header.corr,
6425                    format!("failed to read terminal history: {error}"),
6426                )
6427            })?;
6428        let response = ClientControlResponse::SupervisorTerminals {
6429            module_id,
6430            terminals,
6431        };
6432        Ok(vec![control_response_body_frame(
6433            &frame,
6434            &response,
6435            "ClientControlResponse::SupervisorTerminals",
6436        )?])
6437    }
6438
6439    fn handle_supervisor_routes(
6440        &self,
6441        frame: Frame,
6442        module_id: Option<String>,
6443    ) -> Result<Vec<Frame>, RouterError> {
6444        let modules = self
6445            .forwarding
6446            .route_census(module_id.as_deref())
6447            .map_err(RouterError::Forwarding)?
6448            .into_iter()
6449            .map(|(module_id, routes)| SupervisorRouteModule {
6450                module_id,
6451                routes: routes
6452                    .into_iter()
6453                    .map(|route| SupervisorRoute {
6454                        consumer: match route.principal {
6455                            Principal::Reserved { module_id } => {
6456                                SupervisorRouteConsumer::Reserved { module_id }
6457                            }
6458                            Principal::Direct | Principal::Unverified => {
6459                                SupervisorRouteConsumer::Direct {
6460                                    connection_id: route.goodbye_target.connection_id.get(),
6461                                }
6462                            }
6463                        },
6464                        age_ms: Instant::now()
6465                            .saturating_duration_since(route.bound_at)
6466                            .as_millis()
6467                            .try_into()
6468                            .unwrap_or(u64::MAX),
6469                        draining: route.draining,
6470                        drain_reason: route.drain_reason,
6471                    })
6472                    .collect(),
6473            })
6474            .collect();
6475        let response = ClientControlResponse::SupervisorRoutes { modules };
6476        Ok(vec![control_response_body_frame(
6477            &frame,
6478            &response,
6479            "ClientControlResponse::SupervisorRoutes",
6480        )?])
6481    }
6482
6483    async fn handle_supervisor_provenance(
6484        &self,
6485        ctx: &RouteCtx,
6486        frame: Frame,
6487        module_id: Option<String>,
6488    ) -> Result<Vec<Frame>, RouterError> {
6489        let targeted = module_id.is_some();
6490        let selected = if let Some(module_id) = module_id {
6491            let Some(module) = self.supervisor.get(&module_id) else {
6492                return Ok(vec![control_error_frame(
6493                    &frame,
6494                    "unknown_module",
6495                    format!("module_id '{module_id}' is not supervised"),
6496                )?]);
6497            };
6498            vec![module]
6499        } else {
6500            self.supervisor.list()
6501        };
6502
6503        let target = targeted.then(|| selected[0].module_id());
6504        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6505            let mut modules = Vec::with_capacity(selected.len());
6506            for module in &selected {
6507                let (status, observed_image) = module
6508                    .status_and_running_image_agreement()
6509                    .await
6510                    .map_err(|err| {
6511                        RouterError::backend(
6512                            0,
6513                            frame.header.corr,
6514                            format!("failed to read supervisor status: {err}"),
6515                        )
6516                    })?;
6517                let module_declared = self
6518                    .registry
6519                    .get_module(&status.module_id)
6520                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6521                    .and_then(|registration| registration.manifest.provenance)
6522                    .map(|build| ModuleDeclaredProvenance::Reported { build })
6523                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6524                #[cfg(test)]
6525                let running_image = match &self.provenance_probe_override {
6526                    Some(result) => result.clone(),
6527                    None => observed_image,
6528                };
6529                #[cfg(not(test))]
6530                let running_image = observed_image;
6531                modules.push(SupervisorModuleProvenance {
6532                    module_id: status.module_id,
6533                    module_declared,
6534                    daemon_observed: SupervisorObservedProcess {
6535                        launch_nonce_source: status.launch_nonce_source,
6536                        pid: status.pid,
6537                        spawned_at_ms: status.spawned_at_ms,
6538                        spawned_from: status.spawned_from,
6539                        running_image,
6540                    },
6541                });
6542            }
6543            let daemon = SupervisorDaemonProvenance {
6544                daemon_build: self.daemon_provenance.build.clone(),
6545                daemon_observed: DaemonObservedProcess {
6546                    pid: self.daemon_provenance.pid,
6547                    started_at_ms: self
6548                        .daemon_provenance
6549                        .start_clock
6550                        .map(|clock| clock.started_at_ms())
6551                        .or(self.daemon_provenance.started_at_ms),
6552                    running_image: self
6553                        .daemon_provenance
6554                        .probe
6555                        .observe(
6556                            self.daemon_provenance.pid,
6557                            self.daemon_provenance.executable_path.as_deref(),
6558                            self.daemon_provenance.executable_identity.clone(),
6559                            self.daemon_provenance.process_start_time,
6560                        )
6561                        .await,
6562                },
6563            };
6564            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6565            Ok(vec![control_response_body_frame(
6566                &frame,
6567                &response,
6568                "ClientControlResponse::SupervisorProvenance",
6569            )?])
6570        })
6571        .await
6572    }
6573
6574    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6575        self.refresh_capability_requirements();
6576        let generation = self
6577            .registry
6578            .generation()
6579            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6580        let modules = self
6581            .supervisor
6582            .list()
6583            .into_iter()
6584            .map(|module| {
6585                let status = module.status_for_control("health").map_err(|err| {
6586                    RouterError::backend(
6587                        0,
6588                        frame.header.corr,
6589                        format!("failed to read supervisor health: {err}"),
6590                    )
6591                })?;
6592                let module_id = status.module_id;
6593                let capability_detail = self
6594                    .capability_evaluator
6595                    .required_problem_detail(&module_id);
6596                Ok(SupervisorHealthEntry {
6597                    module_id,
6598                    status: status.health.status,
6599                    detail: append_capability_problem_detail(
6600                        status.health.detail,
6601                        capability_detail,
6602                    ),
6603                    metrics: status.health.metrics,
6604                    consecutive_failures: status.health.consecutive_failures,
6605                    late_answer_count: status.health.late_answer_count,
6606                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6607                    last_action: status.health.last_action,
6608                    last_action_ms: status.health.last_action_ms,
6609                    last_probe_ms: status.health.last_probe_ms,
6610                })
6611            })
6612            .collect::<Result<Vec<_>, RouterError>>()?;
6613        let response = ClientControlResponse::SupervisorHealth {
6614            generation,
6615            modules,
6616        };
6617        Ok(vec![control_response_body_frame(
6618            &frame,
6619            &response,
6620            "ClientControlResponse::SupervisorHealth",
6621        )?])
6622    }
6623
6624    async fn handle_supervisor_restart(
6625        &self,
6626        frame: Frame,
6627        module_id: String,
6628        drain_timeout_ms: Option<u64>,
6629        requester: Principal,
6630    ) -> Result<Vec<Frame>, RouterError> {
6631        let operation_lock = self.supervisor.operation_lock();
6632        let _operation_guard = operation_lock.lock().await;
6633        let Some(module) = self.supervisor.get(&module_id) else {
6634            return Ok(vec![control_error_frame(
6635                &frame,
6636                "unknown_module",
6637                format!("module_id '{module_id}' is not supervised"),
6638            )?]);
6639        };
6640
6641        self.route_outages.mark_operator_action(&module_id);
6642        if let Err(err) = module
6643            .restart_requested(drain_timeout_ms, Some(requester))
6644            .await
6645        {
6646            self.route_outages
6647                .operator_action_ended_unrefused(&module_id);
6648            let (code, message) = match err {
6649                crate::supervise::SuperviseError::Disabled { .. } => {
6650                    ("module_disabled", err.to_string())
6651                }
6652                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6653                    ("swap_in_progress", err.to_string())
6654                }
6655                _ => (
6656                    "target_unavailable",
6657                    format!("failed to restart module_id '{module_id}': {err}"),
6658                ),
6659            };
6660            return Ok(vec![control_error_frame(&frame, code, message)?]);
6661        }
6662
6663        let response = ClientControlResponse::SupervisorAck {
6664            module_id,
6665            applied: true,
6666        };
6667        Ok(vec![control_response_body_frame(
6668            &frame,
6669            &response,
6670            "ClientControlResponse::SupervisorAck",
6671        )?])
6672    }
6673
6674    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
6675    /// when the old process has finished draining: a caller whose own lane
6676    /// rides the old process must get its reply before that drain waits on it.
6677    async fn handle_supervisor_swap(
6678        &self,
6679        frame: Frame,
6680        module_id: String,
6681        ready_timeout_ms: Option<u64>,
6682        requester: Principal,
6683    ) -> Result<Vec<Frame>, RouterError> {
6684        // The daemon-wide operation lock is held only to resolve the handle,
6685        // not across the swap. The swap can take its whole readiness budget,
6686        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
6687        // holding it here would park an operator's stop behind the swap it is
6688        // meant to abort. A rescan or stop that reaches the module during the
6689        // swap is served by the swap itself (see `supervise_swap`).
6690        let module = {
6691            let operation_lock = self.supervisor.operation_lock();
6692            let _operation_guard = operation_lock.lock().await;
6693            self.supervisor.get(&module_id)
6694        };
6695        let Some(module) = module else {
6696            return Ok(vec![control_error_frame(
6697                &frame,
6698                "unknown_module",
6699                format!("module_id '{module_id}' is not supervised"),
6700            )?]);
6701        };
6702
6703        self.route_outages.mark_operator_action(&module_id);
6704        if let Err(err) = module
6705            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6706            .await
6707        {
6708            self.route_outages
6709                .operator_action_ended_unrefused(&module_id);
6710            use crate::supervise::SuperviseError;
6711            let message = err.to_string();
6712            let error = match err {
6713                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6714                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6715                    code: "swap_refused".to_string(),
6716                    message,
6717                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6718                },
6719                SuperviseError::SwapFailed {
6720                    arm,
6721                    candidate_exit,
6722                    ..
6723                } => ErrorBody {
6724                    code: "swap_failed".to_string(),
6725                    message,
6726                    detail: Some(serde_json::json!({
6727                        "arm": arm.as_str(),
6728                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6729                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6730                    })),
6731                },
6732                _ => ErrorBody::new(
6733                    "target_unavailable",
6734                    format!("failed to swap module_id '{module_id}': {message}"),
6735                ),
6736            };
6737            return Ok(vec![control_error_body_frame(&frame, error)?]);
6738        }
6739        // A completed swap kept the incumbent serving until cutover, so it
6740        // usually opened no outage; a mark left behind would make the next,
6741        // unrelated outage read as requested.
6742        self.route_outages
6743            .operator_action_ended_unrefused(&module_id);
6744
6745        let response = ClientControlResponse::SupervisorAck {
6746            module_id,
6747            applied: true,
6748        };
6749        Ok(vec![control_response_body_frame(
6750            &frame,
6751            &response,
6752            "ClientControlResponse::SupervisorAck",
6753        )?])
6754    }
6755
6756    async fn handle_supervisor_reload(
6757        &self,
6758        frame: Frame,
6759        module_id: String,
6760        requester: Principal,
6761    ) -> Result<Vec<Frame>, RouterError> {
6762        let operation_lock = self.supervisor.operation_lock();
6763        let _operation_guard = operation_lock.lock().await;
6764        let Some(module) = self.supervisor.get(&module_id) else {
6765            return Ok(vec![control_error_frame(
6766                &frame,
6767                "unknown_module",
6768                format!("module_id '{module_id}' is not supervised"),
6769            )?]);
6770        };
6771
6772        self.route_outages.mark_operator_action(&module_id);
6773        if let Err(err) = module.reload_requested(Some(requester)).await {
6774            self.route_outages
6775                .operator_action_ended_unrefused(&module_id);
6776            let (code, message) = match err {
6777                crate::supervise::SuperviseError::Disabled { .. } => {
6778                    ("module_disabled", err.to_string())
6779                }
6780                crate::supervise::SuperviseError::SwapInProgress { .. } => {
6781                    ("swap_in_progress", err.to_string())
6782                }
6783                _ => (
6784                    "reload_failed",
6785                    format!("failed to reload module_id '{module_id}': {err}"),
6786                ),
6787            };
6788            return Ok(vec![control_error_frame(&frame, code, message)?]);
6789        }
6790
6791        let response = ClientControlResponse::SupervisorAck {
6792            module_id,
6793            applied: true,
6794        };
6795        Ok(vec![control_response_body_frame(
6796            &frame,
6797            &response,
6798            "ClientControlResponse::SupervisorAck",
6799        )?])
6800    }
6801
6802    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
6803    /// the configuration comes from and whether to apply it; every mode then
6804    /// runs the same comparison, so a preview describes the apply it previews.
6805    async fn handle_supervisor_rescan(
6806        &self,
6807        frame: Frame,
6808        mode: RescanMode,
6809        requester: Principal,
6810    ) -> Result<Vec<Frame>, RouterError> {
6811        let preview = mode.previews();
6812        let Some(context) = self.rescan.clone() else {
6813            return Ok(vec![control_error_frame(
6814                &frame,
6815                "rescan_unavailable",
6816                "the daemon was not started with a reloadable config path".to_string(),
6817            )?]);
6818        };
6819
6820        let operation_lock = self.supervisor.operation_lock();
6821        let _operation_guard = operation_lock.lock().await;
6822        // Loaded under the operation lock, so no other rescan can change the
6823        // staged configuration between a checked apply's comparison and the
6824        // apply itself.
6825        let loaded = match self.load_rescan_config(&mode, &context) {
6826            Ok(config) => config,
6827            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6828        };
6829        // `load` reports a missing file as Ok(None), which is correct at boot
6830        // (no config, nothing to supervise) and catastrophic here: rescan treats
6831        // "not in the config" as "remove it", so an absent file would read as an
6832        // empty module list and retire the entire running fleet. An editor
6833        // writing via write-new-then-rename, or a half-finished edit, is enough
6834        // to open that window. Refuse instead: a config that cannot be read
6835        // carries no instruction to remove anything.
6836        let Some((config, activity_config)) = loaded else {
6837            return Ok(vec![control_error_frame(
6838                &frame,
6839                "invalid_daemon_config",
6840                format!(
6841                    "daemon config not found at {}; refusing to rescan (an absent config would \
6842                     retire every supervised module)",
6843                    context.config_path.display()
6844                ),
6845            )?]);
6846        };
6847        let (
6848            configured_port,
6849            storage_config,
6850            admission_facts_carrier_module_id,
6851            admission_facts_targets,
6852            scope_authority_owners,
6853            federated_scope_owner,
6854            privileged_operator_prompt,
6855            modules,
6856            reserved_capabilities,
6857        ) = (
6858            config.port,
6859            config.storage,
6860            config.admission_facts_carrier_module_id,
6861            config.admission_facts_targets,
6862            config.scope_authority_owners,
6863            config.federated_scope_owner,
6864            config.privileged_operator_prompt,
6865            config.modules,
6866            config.reserved_capabilities,
6867        );
6868
6869        // Collect the sections rescan cannot apply, so the REPLY carries them.
6870        //
6871        // The warning below has always been correct and has always gone only to
6872        // the journal -- addressed to whoever reads logs, while the person who
6873        // just edited the config is looking at the CLI. Naming each section
6874        // individually rather than setting a flag: "something outside modules
6875        // changed" sends the operator back to diffing their own file, which is
6876        // the work this is meant to save.
6877        let mut restart_required = Vec::new();
6878        for section in RestartRequiredSection::ALL {
6879            let changed = match section {
6880                RestartRequiredSection::Port => configured_port != context.configured_port,
6881                RestartRequiredSection::Storage => storage_config != context.storage_config,
6882                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6883                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6884                }
6885                RestartRequiredSection::AdmissionFactsTargets => {
6886                    admission_facts_targets != context.admission_facts_targets
6887                }
6888                RestartRequiredSection::ScopeAuthorityOwners => {
6889                    scope_authority_owners != context.scope_authority_owners
6890                }
6891                RestartRequiredSection::FederatedScopeOwner => {
6892                    federated_scope_owner != context.federated_scope_owner
6893                }
6894                RestartRequiredSection::PrivilegedOperatorPrompt => {
6895                    privileged_operator_prompt != context.privileged_operator_prompt
6896                }
6897            };
6898            if changed {
6899                restart_required.push(section.label().to_string());
6900            }
6901        }
6902        if !restart_required.is_empty() {
6903            warn!(
6904                config_path = %context.config_path.display(),
6905                sections = %restart_required.join(", "),
6906                "daemon config changed outside the modules section; restart the daemon to apply those changes"
6907            );
6908        }
6909
6910        for configured in &modules {
6911            if let Err(err) = validate_spec(&configured.module_spec()) {
6912                return Ok(vec![control_error_frame(
6913                    &frame,
6914                    "invalid_daemon_config",
6915                    format!("supervisor rescan rejected daemon config: {err}"),
6916                )?]);
6917            }
6918        }
6919
6920        let configured_capabilities = modules
6921            .iter()
6922            .map(|module| (module.module_id.clone(), module.enabled))
6923            .collect::<Vec<_>>();
6924        let preview_capability_warnings = if preview {
6925            let (_, registrations) = self.runtime_capability_snapshot()?;
6926            let current_modules = self
6927                .supervisor
6928                .list()
6929                .into_iter()
6930                .map(|module| module.module_id().to_string())
6931                .collect::<BTreeSet<_>>();
6932            let resulting_modules = configured_capabilities.clone();
6933            let removed = current_modules
6934                .into_iter()
6935                .filter(|module_id| {
6936                    !resulting_modules
6937                        .iter()
6938                        .any(|(configured_id, _)| configured_id == module_id)
6939                })
6940                .collect::<Vec<_>>();
6941            // A candidate preview also names providers the candidate disables.
6942            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
6943            let disabled = match &mode {
6944                RescanMode::Candidate(_) => resulting_modules
6945                    .iter()
6946                    .filter(|(_, enabled)| !enabled)
6947                    .map(|(module_id, _)| module_id.clone())
6948                    .collect::<Vec<_>>(),
6949                _ => Vec::new(),
6950            };
6951            self.capability_evaluator.preview_warnings(
6952                resulting_modules,
6953                &removed,
6954                &disabled,
6955                &registrations,
6956            )
6957        } else {
6958            Vec::new()
6959        };
6960        let reconciled = match &mode {
6961            RescanMode::Candidate(candidate) => self
6962                .preview_candidate(&context, modules, candidate)
6963                .map(|result| (result, Vec::new())),
6964            _ => {
6965                self.reconcile_supervised_modules(
6966                    &context.supervisor,
6967                    modules,
6968                    preview,
6969                    requester.clone(),
6970                )
6971                .await
6972            }
6973        };
6974        let (result, changed) = match reconciled {
6975            Ok(result) => result,
6976            Err(message) => {
6977                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6978            }
6979        };
6980        if !preview {
6981            if let Some(ledger) = &self.activity {
6982                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6983                ledger.config_changed(&result.added, &result.removed, &changed, requester);
6984            }
6985            if let Err(code) = self.configure_activity(activity_config).await {
6986                warn!(
6987                    code,
6988                    "activity retention could not be applied during rescan"
6989                );
6990            }
6991            self.capability_evaluator
6992                .configure(configured_capabilities, reserved_capabilities);
6993            self.capability_evaluator.wake_deadline_loop();
6994            self.refresh_capability_requirements();
6995        }
6996        let mut result = result;
6997        result.restart_required = restart_required;
6998        result.capability_warnings = preview_capability_warnings;
6999        let response = ClientControlResponse::SupervisorRescan { result };
7000        Ok(vec![control_response_body_frame(
7001            &frame,
7002            &response,
7003            "ClientControlResponse::SupervisorRescan",
7004        )?])
7005    }
7006
7007    async fn handle_supervisor_release_reserved(
7008        &self,
7009        frame: Frame,
7010        module_id: String,
7011    ) -> Result<Vec<Frame>, RouterError> {
7012        let Some(context) = self.rescan.clone() else {
7013            return Ok(vec![control_error_frame(
7014                &frame,
7015                "release_unavailable",
7016                "reserved-id release requires a daemon started with a reloadable config path",
7017            )?]);
7018        };
7019        let operation_lock = self.supervisor.operation_lock();
7020        let _operation_guard = operation_lock.lock().await;
7021        let loaded = match crate::daemon_config::load(&context.config_path) {
7022            Ok(Some(config)) => config,
7023            Ok(None) => {
7024                return Ok(vec![control_error_frame(
7025                    &frame,
7026                    "invalid_daemon_config",
7027                    format!(
7028                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
7029                        context.config_path.display()
7030                    ),
7031                )?])
7032            }
7033            Err(err) => {
7034                return Ok(vec![control_error_frame(
7035                    &frame,
7036                    "invalid_daemon_config",
7037                    format!("unable to verify reserved-id release against daemon config: {err}"),
7038                )?])
7039            }
7040        };
7041        if loaded
7042            .modules
7043            .iter()
7044            .any(|configured| configured.module_id == module_id)
7045        {
7046            return Ok(vec![control_error_frame(
7047                &frame,
7048                "reserved_module_configured",
7049                format!(
7050                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
7051                ),
7052            )?]);
7053        }
7054        if !self.supervisor.release_retained_reserved_gate(&module_id) {
7055            return Ok(vec![control_error_frame(
7056                &frame,
7057                "reserved_gate_not_retained",
7058                format!(
7059                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
7060                ),
7061            )?]);
7062        }
7063
7064        let response = ClientControlResponse::SupervisorAck {
7065            module_id,
7066            applied: true,
7067        };
7068        Ok(vec![control_response_body_frame(
7069            &frame,
7070            &response,
7071            "ClientControlResponse::SupervisorAck",
7072        )?])
7073    }
7074
7075    /// Compare the configured module set with the supervised one, changing
7076    /// nothing. The staged spec each supervised module holds is what the
7077    /// configured entry is compared with.
7078    fn plan_rescan(
7079        &self,
7080        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7081    ) -> Result<RescanPlan, String> {
7082        let mut current = BTreeMap::new();
7083        for module in self.supervisor.list() {
7084            let (spec, health) = module.configuration().map_err(|err| {
7085                format!(
7086                    "failed to read configuration for module_id '{}': {err}",
7087                    module.module_id()
7088                )
7089            })?;
7090            let enabled = module
7091                .status()
7092                .map_err(|err| {
7093                    format!(
7094                        "failed to read status for module_id '{}': {err}",
7095                        module.module_id()
7096                    )
7097                })?
7098                .enabled;
7099            current.insert(
7100                module.module_id().to_string(),
7101                (module, spec, health, enabled),
7102            );
7103        }
7104        let configured = configured_modules
7105            .into_iter()
7106            .map(|module| (module.module_id.clone(), module))
7107            .collect::<BTreeMap<_, _>>();
7108
7109        let added = configured
7110            .keys()
7111            .filter(|module_id| !current.contains_key(*module_id))
7112            .cloned()
7113            .collect::<Vec<_>>();
7114        let removed = current
7115            .keys()
7116            .filter(|module_id| !configured.contains_key(*module_id))
7117            .cloned()
7118            .collect::<Vec<_>>();
7119        let mut changed_pending_reload = Vec::new();
7120        let mut configuration_changes = BTreeSet::new();
7121        let mut enabled_changes = BTreeSet::new();
7122        let mut unchanged = 0_u32;
7123
7124        for (module_id, configured_module) in &configured {
7125            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
7126            else {
7127                continue;
7128            };
7129            // Compare the whole launch spec so a future launch field cannot
7130            // accidentally become a live-only policy change. Health is stored
7131            // separately and applies live without replacing the process.
7132            let launch_changed = *current_spec != configured_module.module_spec();
7133            let configuration_changed =
7134                launch_changed || *current_health != configured_module.health;
7135            // Applying a rescan also replaces the recorded config value, even
7136            // when an operator already put the runtime in the requested state.
7137            let enabled_changed = *current_enabled != configured_module.enabled
7138                || self
7139                    .capability_evaluator
7140                    .configured_enabled(module_id)
7141                    .is_some_and(|enabled| enabled != configured_module.enabled);
7142            if configuration_changed {
7143                configuration_changes.insert(module_id.clone());
7144            }
7145            if launch_changed {
7146                changed_pending_reload.push(module_id.clone());
7147            }
7148            if enabled_changed {
7149                enabled_changes.insert(module_id.clone());
7150            }
7151            if !configuration_changed && !enabled_changed {
7152                unchanged = unchanged.saturating_add(1);
7153            }
7154        }
7155        Ok(RescanPlan {
7156            current,
7157            configured,
7158            added,
7159            removed,
7160            changed_pending_reload,
7161            configuration_changes,
7162            enabled_changes,
7163            unchanged,
7164        })
7165    }
7166
7167    /// Reconcile the running module set against the configured one.
7168    ///
7169    /// With `preview` set, the diff is computed and returned WITHOUT applying any
7170    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
7171    /// deliberately shares this function with the executing path rather than
7172    /// computing the same diff somewhere else -- two implementations of one
7173    /// decision agree until they do not, and the whole value of a preview is that
7174    /// it describes the operation that will actually run.
7175    async fn reconcile_supervised_modules(
7176        &self,
7177        supervisor: &Supervisor,
7178        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7179        preview: bool,
7180        requester: Principal,
7181    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7182        let plan = self.plan_rescan(configured_modules)?;
7183
7184        // Everything above this point is pure computation over two snapshots.
7185        // Everything below MUTATES. The preview returns here so the boundary is a
7186        // single early return rather than a condition repeated at each mutation
7187        // site, where one missed guard would apply part of a change the caller was
7188        // told would not happen.
7189        if preview {
7190            return Ok((plan.preview_result(), Vec::new()));
7191        }
7192        let changed = plan
7193            .configuration_changes
7194            .union(&plan.enabled_changes)
7195            .cloned()
7196            .collect::<Vec<_>>();
7197        let RescanPlan {
7198            current,
7199            configured,
7200            added,
7201            removed,
7202            changed_pending_reload,
7203            configuration_changes,
7204            enabled_changes,
7205            unchanged,
7206        } = plan;
7207        // The staged generation moves before the first change rather than
7208        // after the last, so a rescan that fails partway still invalidates
7209        // every earlier `supervisor.rescan_checked` preview: part of its diff
7210        // may have run.
7211        if !(added.is_empty()
7212            && removed.is_empty()
7213            && configuration_changes.is_empty()
7214            && enabled_changes.is_empty())
7215        {
7216            if let Some(context) = &self.rescan {
7217                context
7218                    .generation
7219                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7220            }
7221        }
7222
7223        for module_id in &removed {
7224            let module = &current
7225                .get(module_id)
7226                .expect("removed module came from current supervisor state")
7227                .0;
7228            module
7229                .retire_requested(Some(requester.clone()))
7230                .await
7231                .map_err(|err| {
7232                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
7233                })?;
7234            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
7235            //
7236            // `handle_route_open` resolves an absent module in three steps:
7237            // registry, then supervisor status, then tombstone. Retiring first
7238            // opens a window where ALL THREE ARE ABSENT -- the registry entry
7239            // went with the teardown above, the supervisor entry went with
7240            // `retire`, and the tombstone does not exist yet -- so a route.open
7241            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
7242            // it") for a module that was deliberately removed and whose caller
7243            // should get `module_removed` (TERMINAL, carrying a removal age).
7244            //
7245            // Writing the tombstone first closes it: during the window the
7246            // supervisor entry still answers, so the caller gets
7247            // `target_unavailable` -- retryable, and TRUE, because the module
7248            // is mid-teardown. After both statements it is `module_removed`.
7249            // No instant remains where a removed module reads as one that
7250            // never existed.
7251            //
7252            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
7253            // the absence of a test beside a fix invites deletion: these are
7254            // two sync statements with no await between them, so reaching the
7255            // window needs a second worker thread to land exactly between them
7256            // and there is no hook to force it. MEASURED: the 25 daemon_config
7257            // tests pass identically with the old order and the new one, so
7258            // the existing suite cannot see this and a green run is not
7259            // evidence either way. What the suite does hold is the
7260            // post-condition -- a removed module answers `module_removed` --
7261            // which this preserves.
7262            //
7263            // Found by an Athena panel reading the shipped tree against a
7264            // design note (2026-09-19), as the one concrete instance of that
7265            // note's class that survived contact with source. Direction is
7266            // benign: retryable where terminal was intended, never the reverse.
7267            self.supervisor.record_rescan_removal(module_id);
7268            self.supervisor.retire(module_id);
7269            self.route_outages.forget(module_id);
7270        }
7271
7272        for module_id in configured.keys() {
7273            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7274                continue;
7275            };
7276            let configured_module = configured
7277                .get(module_id)
7278                .expect("configured module id came from configured map");
7279            if configuration_changes.contains(module_id) {
7280                module
7281                    .update_configuration(
7282                        configured_module.module_spec(),
7283                        configured_module.health.clone(),
7284                        configured_module.drain_timeout_ms,
7285                    )
7286                    .await
7287                    .map_err(|err| {
7288                        format!(
7289                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
7290                        )
7291                    })?;
7292            }
7293            // Config-only drift is applied by the evaluator's config update;
7294            // do not turn it into a runtime start or stop.
7295            if *current_enabled != configured_module.enabled {
7296                // A rescan that starts or stops a module applies an operator's
7297                // edit to the config, so the resulting outage was asked for.
7298                self.route_outages.mark_operator_action(module_id);
7299                module
7300                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7301                    .await
7302                    .map_err(|err| {
7303                        self.route_outages.operator_action_ended_unrefused(module_id);
7304                        format!(
7305                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7306                            configured_module.enabled
7307                        )
7308                    })?;
7309            }
7310        }
7311
7312        for module_id in &added {
7313            let configured_module = configured
7314                .get(module_id)
7315                .expect("added module id came from configured map");
7316            supervisor
7317                .supervise_configured_requested(
7318                    configured_module.module_spec(),
7319                    configured_module.enabled,
7320                    configured_module.health.clone(),
7321                    configured_module.drain_timeout_ms,
7322                    configured_module.restart,
7323                    configured_module.register_timeout,
7324                    Some(requester.clone()),
7325                )
7326                .map_err(|err| {
7327                    format!("failed to add module_id '{module_id}' during rescan: {err}")
7328                })?;
7329        }
7330
7331        Ok((
7332            SupervisorRescanResult {
7333                added,
7334                removed,
7335                changed_pending_reload,
7336                enabled_changes: enabled_changes.iter().cloned().collect(),
7337                unchanged,
7338                preview: false,
7339                // Filled by the caller, which is the only layer that can see the
7340                // previous config to diff against.
7341                restart_required: Vec::new(),
7342                capability_warnings: Vec::new(),
7343                module_actions: Vec::new(),
7344                earlier_pending: Vec::new(),
7345                expect: None,
7346            },
7347            changed,
7348        ))
7349    }
7350
7351    async fn handle_supervisor_set_enabled(
7352        &self,
7353        frame: Frame,
7354        module_id: String,
7355        enabled: bool,
7356        requester: Principal,
7357    ) -> Result<Vec<Frame>, RouterError> {
7358        let operation_lock = self.supervisor.operation_lock();
7359        let _operation_guard = operation_lock.lock().await;
7360        let Some(module) = self.supervisor.get(&module_id) else {
7361            return Ok(vec![control_error_frame(
7362                &frame,
7363                "unknown_module",
7364                format!("module_id '{module_id}' is not supervised"),
7365            )?]);
7366        };
7367
7368        // Enabling counts as well as disabling: a module an operator starts
7369        // is refused until it registers, and that wait was asked for.
7370        self.route_outages.mark_operator_action(&module_id);
7371        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7372            Ok(applied) => applied,
7373            Err(err) => {
7374                self.route_outages
7375                    .operator_action_ended_unrefused(&module_id);
7376                return Ok(vec![control_error_frame(
7377                    &frame,
7378                    "target_unavailable",
7379                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7380                )?]);
7381            }
7382        };
7383        if !applied {
7384            // Already in the requested state: nothing was made unavailable,
7385            // so the mark must not outlive this request.
7386            self.route_outages
7387                .operator_action_ended_unrefused(&module_id);
7388        }
7389
7390        self.capability_evaluator.wake_deadline_loop();
7391        self.refresh_capability_requirements();
7392        let response = ClientControlResponse::SupervisorAck { module_id, applied };
7393        Ok(vec![control_response_body_frame(
7394            &frame,
7395            &response,
7396            "ClientControlResponse::SupervisorAck",
7397        )?])
7398    }
7399
7400    async fn handle_supervisor_health_probe(
7401        &self,
7402        ctx: &RouteCtx,
7403        frame: Frame,
7404        module_id: String,
7405    ) -> Result<Vec<Frame>, RouterError> {
7406        self.refresh_capability_requirements();
7407        let Some(registration) = self
7408            .registry
7409            .get_module(&module_id)
7410            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7411        else {
7412            return Ok(vec![control_error_frame(
7413                &frame,
7414                "unknown_module",
7415                format!("module_id '{module_id}' is not registered"),
7416            )?]);
7417        };
7418
7419        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
7420        // named for it. Making `module_registration_grants_op` return false
7421        // unconditionally reddens five tests, and every one is named for something
7422        // else -- capability relay, probe/bind demultiplexing, supervision-only
7423        // probing. They exercise a successful advertisement check on the way to their
7424        // own subject.
7425        //
7426        // Real protection, fragile in a specific way: narrowing any of those tests to
7427        // focus on its stated subject would silently remove coverage nobody knows
7428        // they are carrying. Recorded here rather than as a sixth test, because the
7429        // useful fact is WHICH tests hold the guard up -- a new test would add
7430        // coverage without telling the next person what the existing ones quietly do.
7431        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7432        {
7433            return Ok(vec![control_error_frame(
7434                &frame,
7435                "health_not_advertised",
7436                format!("module_id '{module_id}' did not advertise health.check"),
7437            )?]);
7438        }
7439
7440        self.observe_control_wait(
7441            ctx,
7442            Some(&registration.manifest.module_id),
7443            ops::SUPERVISOR_HEALTH_PROBE,
7444            async {
7445                let deadline = Instant::now() + self.health_probe_timeout;
7446                let pending = match self.forwarding.begin_module_control_rpc_for(
7447                    &module_id,
7448                    MODULE_CONTROL_OP_HEALTH_CHECK,
7449                    deadline,
7450                ) {
7451                    Ok(pending) => pending,
7452                    Err(err) => {
7453                        return Ok(vec![control_error_frame(
7454                            &frame,
7455                            forwarding_error_code(&err),
7456                            err.to_string(),
7457                        )?])
7458                    }
7459                };
7460
7461                let PendingModuleControlRpc {
7462                    endpoint,
7463                    module_sink,
7464                    negotiated_ver,
7465                    corr: probe_corr,
7466                    receiver,
7467                } = pending;
7468                let mut guard =
7469                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7470                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7471                    .map_err(|err| {
7472                        RouterError::backend(
7473                            0,
7474                            frame.header.corr,
7475                            format!("failed to encode health.check request: {err}"),
7476                        )
7477                    })?;
7478                let probe_frame = Frame::build_with_version(
7479                    negotiated_ver,
7480                    FrameType::Request,
7481                    control_flags(),
7482                    0,
7483                    0,
7484                    probe_corr,
7485                    probe_body,
7486                )
7487                .map_err(RouterError::FrameBuild)?;
7488
7489                // Queue admission and the reply share one deadline, so a module that
7490                // stops reading cannot park the requesting connection's reader forever.
7491                match timeout_at(deadline, module_sink.send(probe_frame)).await {
7492                    Ok(Ok(())) => {}
7493                    Ok(Err(err)) => {
7494                        return Ok(vec![control_error_frame(
7495                            &frame,
7496                            "target_unavailable",
7497                            err.to_string(),
7498                        )?]);
7499                    }
7500                    Err(_) => {
7501                        return Ok(vec![control_error_frame(
7502                            &frame,
7503                            "module_timeout",
7504                            format!(
7505                                "module_id '{module_id}' did not answer health.check within {:?}",
7506                                self.health_probe_timeout
7507                            ),
7508                        )?]);
7509                    }
7510                }
7511
7512                match timeout_at(deadline, receiver).await {
7513                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7514                        guard.disarm();
7515                        let Some(report) = response.health_report() else {
7516                            return Ok(vec![control_error_frame(
7517                                &frame,
7518                                "invalid_control_body",
7519                                "health.check RPC returned a non-health response",
7520                            )?]);
7521                        };
7522                        // Metrics go out whole here. The supervisor's cached snapshot
7523                        // caps this blob (see truncate_health_metrics), and this path
7524                        // exists precisely to answer without that cap -- so applying it
7525                        // here would leave no way to see what the cached view drops.
7526                        let HealthReport {
7527                            status,
7528                            detail,
7529                            metrics,
7530                        } = report;
7531                        let capability_detail = self
7532                            .capability_evaluator
7533                            .required_problem_detail(&module_id);
7534                        let response = ClientControlResponse::SupervisorHealthProbe {
7535                            module_id,
7536                            status,
7537                            detail: append_capability_problem_detail(detail, capability_detail),
7538                            metrics,
7539                        };
7540                        Ok(vec![control_response_body_frame(
7541                            &frame,
7542                            &response,
7543                            "ClientControlResponse::SupervisorHealthProbe",
7544                        )?])
7545                    }
7546                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7547                        guard.disarm();
7548                        // The module refused the health probe with its own body, so
7549                        // the refusal is relayed without the daemon-origin flag
7550                        // (bit 6): a client must not read it as the daemon's own
7551                        // refusal, which it may treat as never forwarded.
7552                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7553                    }
7554                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7555                        guard.disarm();
7556                        Ok(vec![control_error_frame(
7557                            &frame,
7558                            "target_unavailable",
7559                            message,
7560                        )?])
7561                    }
7562                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7563                        guard.disarm();
7564                        Ok(vec![control_error_frame(
7565                            &frame,
7566                            "invalid_control_body",
7567                            message,
7568                        )?])
7569                    }
7570                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7571                        guard.disarm();
7572                        Ok(vec![control_error_frame(
7573                            &frame,
7574                            "invalid_control_body",
7575                            format!("expected module-control op '{expected}', got '{actual}'"),
7576                        )?])
7577                    }
7578                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7579                        guard.disarm();
7580                        Ok(vec![control_error_frame(
7581                            &frame,
7582                            "module_timeout",
7583                            format!(
7584                                "module_id '{module_id}' answered health.check after {:?}",
7585                                self.health_probe_timeout
7586                            ),
7587                        )?])
7588                    }
7589                    Ok(Err(_)) => Ok(vec![control_error_frame(
7590                        &frame,
7591                        "target_unavailable",
7592                        "health.check waiter was canceled before the module responded",
7593                    )?]),
7594                    Err(_) => Ok(vec![control_error_frame(
7595                        &frame,
7596                        "module_timeout",
7597                        format!(
7598                            "module_id '{module_id}' did not answer health.check within {:?}",
7599                            self.health_probe_timeout
7600                        ),
7601                    )?]),
7602                }
7603            },
7604        )
7605        .await
7606    }
7607
7608    fn supervisor_status(
7609        &self,
7610        module_id: &str,
7611        corr: u64,
7612    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7613        self.supervisor
7614            .get(module_id)
7615            .map(|module| {
7616                let warming = module.is_warming_for_control("status").map_err(|err| {
7617                    RouterError::backend(
7618                        0,
7619                        corr,
7620                        format!(
7621                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
7622                        ),
7623                    )
7624                })?;
7625                module.status_for_control("status").map_err(|err| {
7626                    RouterError::backend(
7627                        0,
7628                        corr,
7629                        format!(
7630                            "failed to read supervisor status for module_id '{module_id}': {err}"
7631                        ),
7632                    )
7633                }).map(|status| (status, warming))
7634            })
7635            .transpose()
7636    }
7637
7638    fn guard_module_control_op(
7639        &self,
7640        frame: &Frame,
7641        module_id: &str,
7642        op: &str,
7643    ) -> Result<Option<Frame>, RouterError> {
7644        if self.module_grants_op(module_id, op, frame.header.corr)? {
7645            return Ok(None);
7646        }
7647
7648        Ok(Some(control_error_frame(
7649            frame,
7650            "op_not_allowed",
7651            format!("module_id '{module_id}' did not grant control op '{op}'"),
7652        )?))
7653    }
7654
7655    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7656        let Some(registration) = self
7657            .registry
7658            .get_module(module_id)
7659            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7660        else {
7661            return Ok(false);
7662        };
7663        Ok(module_registration_grants_op(&registration.control_ops, op))
7664    }
7665
7666    fn handle_status_update(
7667        &self,
7668        endpoint: ModuleEndpointId,
7669        frame: Frame,
7670    ) -> Result<Vec<Frame>, RouterError> {
7671        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7672            Ok(update) => update,
7673            Err(err) => {
7674                // Forward-compat: a newer module may push a channel-0 op this subc
7675                // version doesn't know. The control contract says unknown push ops
7676                // are IGNORED, never answered with an error. Only a malformed body
7677                // for an op we DO know is a real error worth surfacing.
7678                if is_known_module_push_op(&frame.body) {
7679                    return Ok(vec![control_error_frame(
7680                        &frame,
7681                        "invalid_control_body",
7682                        format!("malformed module control push body: {err}"),
7683                    )?]);
7684                }
7685                return Ok(Vec::new());
7686            }
7687        };
7688
7689        match update {
7690            ModuleControlPush::RouteStatus {
7691                route_channel,
7692                route_epoch,
7693                status,
7694            } => {
7695                self.forwarding
7696                    .cache_status(endpoint, route_channel, route_epoch, status)
7697                    .map_err(RouterError::Forwarding)?;
7698            }
7699        }
7700        Ok(Vec::new())
7701    }
7702
7703    fn handle_route_poll(
7704        &self,
7705        ctx: &RouteCtx,
7706        frame: Frame,
7707        route_channel: u16,
7708        route_epoch: u32,
7709        kind: PollKind,
7710    ) -> Result<Vec<Frame>, RouterError> {
7711        let snapshot = self
7712            .forwarding
7713            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7714            .map_err(RouterError::Forwarding)?;
7715        let response = match (kind, snapshot) {
7716            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7717                ClientControlResponse::RoutePoll {
7718                    route_channel,
7719                    route_epoch,
7720                    status,
7721                    live: None,
7722                }
7723            }
7724            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7725                route_channel,
7726                route_epoch,
7727                status: None,
7728                live: None,
7729            },
7730            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7731                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
7732                // is what makes reporting `true` correct rather than a
7733                // confident guess. `process_live` returns None only when the
7734                // module id has no supervisor snapshot at all -- an
7735                // externally-started module the daemon did not spawn -- and
7736                // for those the supervisor has no opinion to offer, ever. It
7737                // is never None for a supervised module in an unknown state:
7738                // a supervised module always has a snapshot, and the answer
7739                // comes from `state == Running && process_alive`.
7740                //
7741                // The route is Bound, so the module completed a HELLO on a
7742                // live connection; "the process this route points at is
7743                // running" is therefore attested by the binding rather than
7744                // assumed. Reporting `false` for an unsupervised module would
7745                // be the actual lie -- it would tell a client its healthy
7746                // route is dead because the daemon does not manage the
7747                // process.
7748                //
7749                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
7750                // module whose liveness is genuinely unknown, e.g. a snapshot
7751                // that has not been populated yet -- THIS DEFAULT BECOMES
7752                // WRONG and must split: unsupervised stays true, unknown
7753                // becomes null so the client can tell the two apart. The
7754                // response field is already `Option<bool>`, so the wire can
7755                // carry that distinction today.
7756                let live = self
7757                    .process_liveness
7758                    .as_ref()
7759                    .and_then(|source| source.process_live(&module_id))
7760                    .unwrap_or(true);
7761                ClientControlResponse::RoutePoll {
7762                    route_channel,
7763                    route_epoch,
7764                    status: None,
7765                    live: Some(live),
7766                }
7767            }
7768            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7769                route_channel,
7770                route_epoch,
7771                status: None,
7772                live: Some(false),
7773            },
7774        };
7775
7776        Ok(vec![control_response_body_frame(
7777            &frame,
7778            &response,
7779            "ClientControlResponse::RoutePoll",
7780        )?])
7781    }
7782
7783    pub(crate) fn observe_module_control_completion(
7784        &self,
7785        completion: ModuleControlRpcCompletion,
7786    ) -> bool {
7787        match completion {
7788            ModuleControlRpcCompletion::Unknown => false,
7789            ModuleControlRpcCompletion::Settled => true,
7790            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7791                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7792                info!(
7793                    module_id = %module_id,
7794                    latency_ms,
7795                    "late health.check answer proves the module is alive"
7796                );
7797                match self
7798                    .supervisor
7799                    .record_late_health_answer(&module_id, latency_ms)
7800                {
7801                    Ok(true) => {}
7802                    Ok(false) => debug!(
7803                        module_id = %module_id,
7804                        latency_ms,
7805                        "late health.check answer has no active supervisor snapshot"
7806                    ),
7807                    Err(err) => warn!(
7808                        module_id = %module_id,
7809                        latency_ms,
7810                        error = %err,
7811                        "failed to record late health.check answer"
7812                    ),
7813                }
7814                true
7815            }
7816        }
7817    }
7818
7819    /// Decide whether a failure while settling a relayed `route.bind` belongs to
7820    /// the module connection whose frame is being handled, or to the client that
7821    /// relay was opened for.
7822    ///
7823    /// This runs on the MODULE connection's frame handler, where returning `Err`
7824    /// ends that connection -- and a module connection carries every client's
7825    /// routes to that module, so ending it costs the whole fleet its tools.
7826    /// `ConnectionClosing` carries the id of the connection that is closing, and
7827    /// when that id is a CLIENT's, the condition is entirely about that one
7828    /// client's route.open. A client-scoped condition has no authority over a
7829    /// shared module connection, so it is logged and the single relay is dropped:
7830    /// the client is going away, and `complete_pending_relay` already removed the
7831    /// relay before failing, so there is nothing left to settle. Anything that
7832    /// relay still reserved is released by that client's own connection teardown,
7833    /// which is already under way -- that is what "closing" means.
7834    ///
7835    /// Every other failure is a statement about THIS connection and stays fatal:
7836    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
7837    /// id in `ConnectionClosing` all mean this connection cannot keep serving
7838    /// frames correctly.
7839    fn refuse_to_end_module_connection_for_a_client(
7840        &self,
7841        module_connection_id: ConnectionId,
7842        corr: u64,
7843        err: ForwardingError,
7844    ) -> Result<(), RouterError> {
7845        if let ForwardingError::ConnectionClosing { connection_id } = err {
7846            if connection_id != module_connection_id {
7847                warn!(
7848                    module_connection_id = module_connection_id.get(),
7849                    client_connection_id = connection_id.get(),
7850                    corr,
7851                    "dropping a route.bind response for a closing client; the module connection keeps serving"
7852                );
7853                return Ok(());
7854            }
7855        }
7856        Err(RouterError::Forwarding(err))
7857    }
7858
7859    fn handle_module_relay_response(
7860        &self,
7861        connection_id: ConnectionId,
7862        frame: Frame,
7863    ) -> Result<Vec<Frame>, RouterError> {
7864        let mut secondary_error = None;
7865        let outcome = match frame.header.ty {
7866            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7867                Ok(probe) if probe.op == "route.bind" => {
7868                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7869                        Ok(ModuleControlResponse::RouteBindAck {}) => {
7870                            RouteBindRelayOutcome::Accepted
7871                        }
7872                        Ok(other) => {
7873                            let message =
7874                                format!("route.bind response carried unexpected body: {other:?}");
7875                            secondary_error = Some(control_error_frame(
7876                                &frame,
7877                                "invalid_control_body",
7878                                message.clone(),
7879                            )?);
7880                            RouteBindRelayOutcome::ModuleGone(message)
7881                        }
7882                        Err(err) => {
7883                            let message = format!("malformed route.bind response body: {err}");
7884                            secondary_error = Some(control_error_frame(
7885                                &frame,
7886                                "invalid_control_body",
7887                                message.clone(),
7888                            )?);
7889                            RouteBindRelayOutcome::ModuleGone(message)
7890                        }
7891                    }
7892                }
7893                Ok(probe) => {
7894                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7895                    {
7896                        Ok(response) => ModuleControlRpcOutcome::Response(response),
7897                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7898                            "malformed {} response body: {err}",
7899                            probe.op
7900                        )),
7901                    };
7902                    let completion = self
7903                        .forwarding
7904                        .complete_module_control_rpc(
7905                            connection_id,
7906                            frame.header.corr,
7907                            Some(&probe.op),
7908                            outcome,
7909                        )
7910                        .map_err(RouterError::Forwarding)?;
7911                    if !self.observe_module_control_completion(completion) {
7912                        debug!(
7913                            connection_id = connection_id.get(),
7914                            corr = frame.header.corr,
7915                            op = %probe.op,
7916                            "dropping late or unknown module-control RPC response"
7917                        );
7918                    }
7919                    return Ok(Vec::new());
7920                }
7921                Err(err) => {
7922                    if let Some(expected_op) = self
7923                        .forwarding
7924                        .pending_module_control_op(connection_id, frame.header.corr)
7925                        .map_err(RouterError::Forwarding)?
7926                    {
7927                        let completion = self
7928                            .forwarding
7929                            .complete_module_control_rpc(
7930                                connection_id,
7931                                frame.header.corr,
7932                                None,
7933                                ModuleControlRpcOutcome::MalformedResponse(format!(
7934                                    "malformed {expected_op} response body: {err}"
7935                                )),
7936                            )
7937                            .map_err(RouterError::Forwarding)?;
7938                        if !self.observe_module_control_completion(completion) {
7939                            debug!(
7940                                connection_id = connection_id.get(),
7941                                corr = frame.header.corr,
7942                                "dropping late malformed module-control RPC response"
7943                            );
7944                        }
7945                        return Ok(Vec::new());
7946                    }
7947                    let message = format!("malformed route.bind response body: {err}");
7948                    secondary_error = Some(control_error_frame(
7949                        &frame,
7950                        "invalid_control_body",
7951                        message.clone(),
7952                    )?);
7953                    RouteBindRelayOutcome::ModuleGone(message)
7954                }
7955            },
7956            FrameType::Error => {
7957                if self
7958                    .forwarding
7959                    .pending_module_control_op(connection_id, frame.header.corr)
7960                    .map_err(RouterError::Forwarding)?
7961                    .is_some()
7962                {
7963                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7964                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7965                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7966                            "malformed module-control ERROR body: {err}"
7967                        )),
7968                    };
7969                    let completion = self
7970                        .forwarding
7971                        .complete_module_control_rpc(
7972                            connection_id,
7973                            frame.header.corr,
7974                            None,
7975                            outcome,
7976                        )
7977                        .map_err(RouterError::Forwarding)?;
7978                    if !self.observe_module_control_completion(completion) {
7979                        debug!(
7980                            connection_id = connection_id.get(),
7981                            corr = frame.header.corr,
7982                            "dropping late or unknown module-control RPC error"
7983                        );
7984                    }
7985                    return Ok(Vec::new());
7986                }
7987                match serde_json::from_slice::<ErrorBody>(&frame.body) {
7988                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
7989                    Err(err) => {
7990                        let message = format!("malformed route.bind ERROR body: {err}");
7991                        secondary_error = Some(control_error_frame(
7992                            &frame,
7993                            "invalid_control_body",
7994                            message.clone(),
7995                        )?);
7996                        RouteBindRelayOutcome::ModuleGone(message)
7997                    }
7998                }
7999            }
8000            ty => {
8001                return Ok(vec![control_error_frame(
8002                    &frame,
8003                    "unsupported_control_frame",
8004                    format!("unsupported module channel-0 frame {ty:?}"),
8005                )?])
8006            }
8007        };
8008
8009        let settled =
8010            self.forwarding
8011                .complete_pending_relay(connection_id, frame.header.corr, outcome);
8012        let completion = match settled {
8013            Ok(completion) => completion,
8014            Err(err) => {
8015                self.refuse_to_end_module_connection_for_a_client(
8016                    connection_id,
8017                    frame.header.corr,
8018                    err,
8019                )?;
8020                return Ok(secondary_error.into_iter().collect());
8021            }
8022        };
8023        if let Some(target) = completion.abandoned.as_ref() {
8024            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
8025        }
8026        if !completion.settled {
8027            debug!(
8028                connection_id = connection_id.get(),
8029                corr = frame.header.corr,
8030                frame_type = ?frame.header.ty,
8031                "dropping late or unknown route.bind relay response"
8032            );
8033        }
8034        Ok(secondary_error.into_iter().collect())
8035    }
8036
8037    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
8038        debug!(connection_id = connection_id.get(), "handling GOODBYE");
8039        // GOODBYE ends the connection's logical session even when its socket
8040        // stays open. Use disconnect teardown so verdicts, client notices and
8041        // scope authority are released at the same lifecycle boundary.
8042        self.cleanup_connection_with_end_reason(
8043            connection_id,
8044            RegistrationEndReason::ExplicitGoodbye,
8045        )
8046        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
8047        Ok(Vec::new())
8048    }
8049}
8050
8051impl Default for ControlHandler {
8052    fn default() -> Self {
8053        Self::new(Arc::new(Registry::default()))
8054    }
8055}
8056
8057impl crate::supervise::SwapPromotionObserver for ControlHandler {
8058    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
8059        self.apply_registration_capabilities(registration);
8060    }
8061}
8062
8063fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
8064    CapabilityRequirementStatus {
8065        consumer: status.consumer,
8066        capability: status.capability,
8067        need: match status.need {
8068            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
8069            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
8070        },
8071        verdict: status.verdict.as_str().to_string(),
8072        episode_seq: status.episode_seq,
8073        config_satisfiable: status.config_satisfiable,
8074        runtime_available: status.runtime_available,
8075        detail: status.detail,
8076    }
8077}
8078
8079fn append_capability_problem_detail(
8080    detail: Option<String>,
8081    capability_detail: Option<String>,
8082) -> Option<String> {
8083    match (detail, capability_detail) {
8084        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
8085        (Some(detail), None) => Some(detail),
8086        (None, Some(capability_detail)) => Some(capability_detail),
8087        (None, None) => None,
8088    }
8089}
8090
8091fn subc_ops() -> Vec<String> {
8092    SUBC_CONTROL_OPS
8093        .iter()
8094        .map(|op| (*op).to_string())
8095        .collect()
8096}
8097
8098fn module_subc_ops() -> Vec<String> {
8099    SUBC_CONTROL_OPS
8100        .iter()
8101        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
8102        .map(|op| (*op).to_string())
8103        .collect()
8104}
8105
8106#[cfg(test)]
8107fn module_baseline_control_ops() -> Vec<String> {
8108    MODULE_BASELINE_CONTROL_OPS
8109        .iter()
8110        .map(|op| (*op).to_string())
8111        .collect()
8112}
8113
8114fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
8115    let mut seen = HashSet::new();
8116    let mut effective = Vec::new();
8117    for op in MODULE_BASELINE_CONTROL_OPS {
8118        if seen.insert((*op).to_string()) {
8119            effective.push((*op).to_string());
8120        }
8121    }
8122    for op in declared.unwrap_or_default() {
8123        if seen.insert(op.clone()) {
8124            effective.push(op);
8125        }
8126    }
8127    effective
8128}
8129
8130fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8131    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8132}
8133
8134fn target_module_id(target: &RouteTarget) -> &str {
8135    match target {
8136        RouteTarget::ToolProvider { module_id }
8137        | RouteTarget::ManagementSurface { module_id }
8138        | RouteTarget::InternalService { module_id, .. } => module_id,
8139    }
8140}
8141
8142fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8143    roles.iter().any(|role| match (target, role) {
8144        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8145        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8146        (
8147            RouteTarget::InternalService { service_id, .. },
8148            ProviderRole::InternalService {
8149                service_id: provided,
8150                ..
8151            },
8152        ) => service_id == provided,
8153        _ => false,
8154    })
8155}
8156
8157fn is_routable_role(role: &ProviderRole) -> bool {
8158    matches!(
8159        role,
8160        ProviderRole::ToolProvider { .. }
8161            | ProviderRole::ManagementSurface { .. }
8162            | ProviderRole::InternalService { .. }
8163    )
8164}
8165
8166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8167enum ControlRequestBodyError {
8168    UnknownOp,
8169    InvalidBody,
8170}
8171
8172#[derive(Debug, Deserialize)]
8173struct ControlOpProbe {
8174    op: String,
8175}
8176
8177/// Channel-0 push ops this subc version understands. A push whose `op` is not in
8178/// this set is treated as a forward-compat unknown and ignored rather than errored.
8179const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8180
8181fn is_known_module_push_op(body: &[u8]) -> bool {
8182    serde_json::from_slice::<ControlOpProbe>(body)
8183        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8184        .unwrap_or(false)
8185}
8186
8187fn is_known_module_request_op(body: &[u8]) -> bool {
8188    serde_json::from_slice::<ControlOpProbe>(body)
8189        .map(|probe| is_module_to_subc_op(&probe.op))
8190        .unwrap_or(false)
8191}
8192
8193fn is_module_to_subc_op(op: &str) -> bool {
8194    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8195}
8196
8197fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8198    debug!(
8199        op = %op,
8200        connection_id = connection_id.get(),
8201        corr,
8202        "control dispatch"
8203    );
8204}
8205
8206fn log_slow_control_dispatch(
8207    dispatch_started_at: Option<StdInstant>,
8208    op: &'static str,
8209    connection_id: ConnectionId,
8210    corr: u64,
8211    route_open_timings: Option<&RouteOpenTimings>,
8212) {
8213    let Some(dispatch_started_at) = dispatch_started_at else {
8214        return;
8215    };
8216    // Use the same monotonic clock as the phase timers and relay deadline.
8217    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8218    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8219        if let Some(timings) = route_open_timings {
8220            warn!(
8221                op = %op,
8222                connection_id = connection_id.get(),
8223                corr,
8224                elapsed_ms = elapsed.as_millis() as u64,
8225                module_id = ?timings.module_id,
8226                principal = %timings.principal,
8227                admission_ms = timings.admission.as_millis() as u64,
8228                project_root_ms = timings.project_root.as_millis() as u64,
8229                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8230                bind_send_ms = timings.bind_send.as_millis() as u64,
8231                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8232                "slow control dispatch"
8233            );
8234            return;
8235        }
8236        warn!(
8237            op = %op,
8238            connection_id = connection_id.get(),
8239            corr,
8240            elapsed_ms = elapsed.as_millis() as u64,
8241            "slow control dispatch"
8242        );
8243    }
8244}
8245
8246/// A test stand-in for the OS caller lookup, given the recorded endpoints.
8247#[cfg(test)]
8248type TestCallerLookup =
8249    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8250
8251/// Every new client operation needs an explicit privilege decision here.
8252fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8253    match request {
8254        ClientControlRequest::SupervisorRestart { module_id, .. } => {
8255            Some(PrivilegedOperation::Restart { module_id })
8256        }
8257        ClientControlRequest::SupervisorSwap { module_id, .. } => {
8258            Some(PrivilegedOperation::Swap { module_id })
8259        }
8260        ClientControlRequest::SupervisorReload { module_id } => {
8261            Some(PrivilegedOperation::Reload { module_id })
8262        }
8263        ClientControlRequest::SupervisorRescan { preview } => {
8264            if *preview {
8265                None
8266            } else {
8267                Some(PrivilegedOperation::Rescan)
8268            }
8269        }
8270        // Only the well-formed apply asks for approval. A preview changes
8271        // nothing, and any other combination is refused before it touches
8272        // anything, so prompting the operator for it would ask about a request
8273        // that cannot run.
8274        ClientControlRequest::SupervisorRescanChecked {
8275            preview,
8276            candidate,
8277            expect,
8278        } => (!*preview && candidate.is_none() && expect.is_some())
8279            .then_some(PrivilegedOperation::Rescan),
8280        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8281            Some(PrivilegedOperation::SetEnabled {
8282                module_id,
8283                enabled: *enabled,
8284            })
8285        }
8286        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8287            Some(PrivilegedOperation::ReleaseReserved { module_id })
8288        }
8289        ClientControlRequest::ServerDescribe {}
8290        // The announcement uses the same client admission as supervisor.restart,
8291        // but cannot stop anything itself and needs no operator confirmation.
8292        | ClientControlRequest::DaemonShutdownIntent { .. }
8293        | ClientControlRequest::CatalogList { .. }
8294        | ClientControlRequest::RouteOpen { .. }
8295        | ClientControlRequest::RoutePoll { .. }
8296        | ClientControlRequest::SupervisorList {}
8297        | ClientControlRequest::SupervisorHealth {}
8298        | ClientControlRequest::SupervisorHealthProbe { .. }
8299        | ClientControlRequest::SupervisorRoutes { .. }
8300        | ClientControlRequest::SupervisorProvenance { .. }
8301        | ClientControlRequest::SupervisorStderrTail { .. }
8302        | ClientControlRequest::SupervisorTerminals { .. }
8303        | ClientControlRequest::SupervisorSpawnSnapshot {}
8304        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8305    }
8306}
8307
8308fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8309    match request {
8310        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8311        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8312        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8313        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8314        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8315        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8316        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8317        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8318        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8319        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8320        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8321        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8322        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8323        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8324        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8325        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8326        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8327        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8328        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8329        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8330        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8331    }
8332}
8333
8334/// The caller of one `route.open`, as its `route.open refused` line names it.
8335///
8336/// A connection id alone cannot name a caller after the fact: ids restart at
8337/// every daemon restart, so the same number belongs to different modules
8338/// before and after one. These three fields are what the refused line needs
8339/// to be attributable on its own, and they are rendered with the same helpers
8340/// as the `route.open accepted` line so both directions grep alike.
8341struct RouteOpenCaller {
8342    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
8343    /// when the request presented a consumer identity whose launch nonce did
8344    /// not verify. The claimed module id is never printed as a principal.
8345    principal: String,
8346    /// `none` for an unscoped open; the admitted stamp's label once scope
8347    /// admission issued one; before that, the requested selector's label.
8348    scope: String,
8349    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
8350    /// because it is a string the caller chose.
8351    harness: String,
8352}
8353
8354impl RouteOpenCaller {
8355    fn new(
8356        principal: Option<&Principal>,
8357        scope: Option<&ScopeSelector>,
8358        identity: &BindIdentity,
8359    ) -> Self {
8360        Self {
8361            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8362            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8363            harness: bounded_log_text(&identity.harness),
8364        }
8365    }
8366
8367    /// For a refusal that has no parsed request at hand: names whatever the
8368    /// frame still says, and `unknown` where it says nothing parseable.
8369    fn unknown() -> Self {
8370        Self {
8371            principal: "unknown".to_string(),
8372            scope: "unknown".to_string(),
8373            harness: "unknown".to_string(),
8374        }
8375    }
8376}
8377
8378/// Longest client-chosen text (a scope ref, a harness) a route.open log line
8379/// prints before cutting it.
8380const MAX_LOGGED_TEXT_CHARS: usize = 64;
8381
8382/// `value` Debug-quoted, so control characters land escaped and cannot forge
8383/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
8384/// `…` inside the quotes so one request cannot flood the line.
8385fn bounded_log_text(value: &str) -> String {
8386    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8387    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8388        bounded.push('…');
8389    }
8390    format!("{bounded:?}")
8391}
8392
8393/// The `scope=` field of the `route.open accepted` line, from the stamp that
8394/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
8395/// so a ref holding control characters cannot forge a log line, and cut to 64
8396/// characters with a trailing `…` to bound the line. The bind proof and the
8397/// scope's attributes never appear.
8398fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8399    use subc_protocol::scope::ScopeKind;
8400    let kind = match stamp.kind {
8401        ScopeKind::Head => "head",
8402        ScopeKind::Worker => "worker",
8403        ScopeKind::Ephemeral => "ephemeral",
8404        ScopeKind::Federated => "federated",
8405        _ => "unknown",
8406    };
8407    format!(
8408        "{kind}/{}/{}@{}",
8409        principal_label(&stamp.owner),
8410        bounded_log_text(&stamp.scope_ref),
8411        stamp.scope_epoch
8412    )
8413}
8414
8415/// The `scope=` field of a `route.open refused` line for an open refused
8416/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
8417/// The kind is not known yet, so the literal `requested` stands in its place.
8418/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
8419/// owner it has not been checked against the configured modules: a quoted
8420/// owner means the value was never validated. The bind proof never appears.
8421fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8422    let epoch = selector
8423        .scope_epoch
8424        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8425    format!(
8426        "requested/{}/{}@{epoch}",
8427        bounded_log_text(&principal_label(&selector.owner)),
8428        bounded_log_text(&selector.scope_ref),
8429    )
8430}
8431
8432fn principal_label(principal: &Principal) -> String {
8433    match principal {
8434        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8435        Principal::Direct => "direct".to_string(),
8436        other => format!("{other:?}"),
8437    }
8438}
8439
8440fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8441    match request {
8442        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8443        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8444        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8445        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8446        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8447        ModuleControlRequestFromModule::RouteAuthorize { .. } => {
8448            subc_protocol::scope::ROUTE_AUTHORIZE_OP
8449        }
8450        ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8451        _ => "unsupported",
8452    }
8453}
8454
8455fn parse_client_control_request(
8456    body: &[u8],
8457) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8458    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8459        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8460            Ok(probe)
8461                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8462                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8463            {
8464                ControlRequestBodyError::InvalidBody
8465            }
8466            Ok(_) => ControlRequestBodyError::UnknownOp,
8467            Err(_) => ControlRequestBodyError::InvalidBody,
8468        };
8469        (err, classification)
8470    })
8471}
8472
8473fn parse_module_control_request_from_module(
8474    body: &[u8],
8475) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8476    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8477        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8478            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8479            Ok(_) => ControlRequestBodyError::UnknownOp,
8480            Err(_) => ControlRequestBodyError::InvalidBody,
8481        };
8482        (err, classification)
8483    })
8484}
8485
8486#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8487enum ProviderRoleKind {
8488    ToolProvider,
8489    PipelineStage,
8490    ManagementSurface,
8491    InternalService,
8492}
8493
8494fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8495    match role {
8496        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8497        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8498        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8499        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8500    }
8501}
8502
8503fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8504    roles.iter().map(provider_role_kind).collect()
8505}
8506
8507/// Most refused scope records named individually in the log per sync; the
8508/// `refused` count on the accepted line is always complete.
8509const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8510
8511/// Per-outcome counts of one accepted `scope.sync`, for its log line.
8512#[derive(Debug, Default, PartialEq, Eq)]
8513struct ScopeOutcomeCounts {
8514    created: usize,
8515    replaced: usize,
8516    updated: usize,
8517    unchanged: usize,
8518    refused: usize,
8519}
8520
8521impl ScopeOutcomeCounts {
8522    fn of(results: &[ScopeRecordResult]) -> Self {
8523        let mut counts = Self::default();
8524        for result in results {
8525            let slot = match result.outcome {
8526                ScopeRecordOutcome::Created => &mut counts.created,
8527                ScopeRecordOutcome::Replaced => &mut counts.replaced,
8528                ScopeRecordOutcome::Updated => &mut counts.updated,
8529                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8530                ScopeRecordOutcome::Refused => &mut counts.refused,
8531            };
8532            *slot += 1;
8533        }
8534        counts
8535    }
8536}
8537
8538#[cfg(test)]
8539mod scope_outcome_count_tests {
8540    use super::*;
8541
8542    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8543        ScopeRecordResult {
8544            scope_ref: "r".to_string(),
8545            scope_epoch: 1,
8546            outcome,
8547            code: None,
8548            message: None,
8549            version: None,
8550            parent_state: None,
8551        }
8552    }
8553
8554    /// Each outcome lands in its own count, so a refused record can never be
8555    /// hidden inside the total the log already printed.
8556    #[test]
8557    fn every_outcome_is_counted_in_its_own_field() {
8558        let results = [
8559            result(ScopeRecordOutcome::Created),
8560            result(ScopeRecordOutcome::Created),
8561            result(ScopeRecordOutcome::Replaced),
8562            result(ScopeRecordOutcome::Updated),
8563            result(ScopeRecordOutcome::Unchanged),
8564            result(ScopeRecordOutcome::Refused),
8565            result(ScopeRecordOutcome::Refused),
8566            result(ScopeRecordOutcome::Refused),
8567        ];
8568        assert_eq!(
8569            ScopeOutcomeCounts::of(&results),
8570            ScopeOutcomeCounts {
8571                created: 2,
8572                replaced: 1,
8573                updated: 1,
8574                unchanged: 1,
8575                refused: 3,
8576            }
8577        );
8578    }
8579}
8580
8581/// Return whether a catalog change can create a newly violating live route.
8582/// Removing an attested claim is intentionally excluded: it makes fewer routes
8583/// forbidden and therefore must leave the existing route census untouched.
8584fn capability_census_trigger(
8585    old: Option<&CapabilityDeclarations>,
8586    new: Option<&CapabilityDeclarations>,
8587) -> bool {
8588    let old_provides = old
8589        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8590        .unwrap_or_default();
8591    let old_denies = old
8592        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8593        .unwrap_or_default();
8594    let new = new.cloned().unwrap_or(CapabilityDeclarations {
8595        provides: Vec::new(),
8596        requires: Vec::new(),
8597        must_never_reach: Vec::new(),
8598    });
8599
8600    new.provides
8601        .iter()
8602        .any(|capability| !old_provides.contains(capability))
8603        || new
8604            .must_never_reach
8605            .iter()
8606            .any(|capability| !old_denies.contains(capability))
8607}
8608
8609fn catalog_update_frozen_field_message(
8610    registered: &ModuleManifest,
8611    provides: &[ProviderRole],
8612) -> Option<String> {
8613    let old_has_provides = !registered.provides.is_empty();
8614    let new_has_provides = !provides.is_empty();
8615    if old_has_provides != new_has_provides {
8616        return Some(format!(
8617            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8618            registered.module_id
8619        ));
8620    }
8621
8622    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
8623        return Some(format!(
8624            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8625            registered.module_id
8626        ));
8627    }
8628
8629    let registered_concurrency = manifest_concurrency(registered);
8630    let mut candidate = registered.clone();
8631    candidate.provides = provides.to_vec();
8632    let candidate_concurrency = manifest_concurrency(&candidate);
8633    if candidate_concurrency != registered_concurrency {
8634        return Some(format!(
8635            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8636            registered.module_id, registered_concurrency, candidate_concurrency
8637        ));
8638    }
8639
8640    // control_ops live beside the manifest in the HELLO body, not inside
8641    // ModuleManifest, so a provides-only catalog.update cannot change them.
8642    None
8643}
8644
8645fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8646    manifest.provides.iter().any(is_routable_role)
8647}
8648
8649/// Returns the routable-provider concurrency subc should enforce for this manifest.
8650///
8651/// ToolProvider and ManagementSurface store their delivery concurrency directly.
8652/// InternalService has no role-specific concurrency field, so it retains the
8653/// existing ModuleManaged default for backward compatibility.
8654fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8655    manifest
8656        .provides
8657        .iter()
8658        .find_map(|provider| match provider {
8659            ProviderRole::ToolProvider { concurrency, .. }
8660            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8661            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8662        })
8663        .unwrap_or(Concurrency::ModuleManaged)
8664}
8665
8666/// True when the manifest carries a ManagementSurface role whose concurrency
8667/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
8668/// bytes because the typed manifest deliberately erases that distinction: the
8669/// default exists for wire compatibility, and this probe exists so the default
8670/// stays observable. Any parse irregularity returns false -- the caller only
8671/// logs, and a malformed body already failed registration upstream.
8672fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8673    let has_management_surface = manifest
8674        .provides
8675        .iter()
8676        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8677    if !has_management_surface {
8678        return false;
8679    }
8680    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8681        return false;
8682    };
8683    let Some(provides) = raw
8684        .get("manifest")
8685        .and_then(|manifest| manifest.get("provides"))
8686        .and_then(serde_json::Value::as_array)
8687    else {
8688        return false;
8689    };
8690    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
8691    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
8692    // against the management_surface_manifest_without_concurrency golden, not
8693    // recalled (the externally-tagged guess was this function's first bug).
8694    provides.iter().any(|role| {
8695        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8696            && role.get("concurrency").is_none()
8697    })
8698}
8699
8700fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8701    if peer_version != PROTOCOL_VERSION {
8702        return Err(format!(
8703            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8704        ));
8705    }
8706    Ok(PROTOCOL_VERSION)
8707}
8708
8709fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8710    Frame::build_with_version(
8711        response_version(frame),
8712        FrameType::Pong,
8713        frame.header.flags,
8714        0,
8715        0,
8716        frame.header.corr,
8717        Vec::new(),
8718    )
8719    .map_err(RouterError::FrameBuild)
8720}
8721
8722pub(crate) fn control_error_frame(
8723    frame: &Frame,
8724    code: &'static str,
8725    message: impl Into<String>,
8726) -> Result<Frame, RouterError> {
8727    control_error_body_frame(
8728        frame,
8729        ErrorBody {
8730            code: code.to_string(),
8731            message: message.into(),
8732            detail: None,
8733        },
8734    )
8735}
8736
8737/// A channel-0 ERROR for a refusal the daemon decided itself: every
8738/// route.open refusal, the drain `module_reloading` and scope-change/end
8739/// bodies, spawn-subscription and supervisor refusals. Carries the
8740/// daemon-origin bit, which is what lets a client treat `module_reloading`
8741/// as proof that its request never reached a module. Response, push,
8742/// goodbye and relay-request frames keep the plain control flags in their
8743/// own builders.
8744fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8745    control_error_frame_with_flags(frame, error, daemon_error_flags())
8746}
8747
8748/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
8749/// refused route.bind answered as a route.open refusal, or a refused
8750/// module-control RPC. Plain control flags — the code and message are
8751/// module-chosen, so the frame must not claim daemon authorship, even when
8752/// the module picks a daemon code like `module_reloading`.
8753fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8754    control_error_frame_with_flags(frame, error, control_flags())
8755}
8756
8757fn control_error_frame_with_flags(
8758    frame: &Frame,
8759    error: ErrorBody,
8760    flags: Flags,
8761) -> Result<Frame, RouterError> {
8762    let body = serde_json::to_vec(&error).map_err(|err| {
8763        RouterError::backend(
8764            0,
8765            frame.header.corr,
8766            format!("failed to encode control ERROR: {err}"),
8767        )
8768    })?;
8769
8770    Frame::build_with_version(
8771        response_version(frame),
8772        FrameType::Error,
8773        flags,
8774        0,
8775        0,
8776        frame.header.corr,
8777        body,
8778    )
8779    .map_err(RouterError::FrameBuild)
8780}
8781
8782pub(crate) fn control_response_body_frame<T: Serialize>(
8783    frame: &Frame,
8784    reply: &T,
8785    label: &'static str,
8786) -> Result<Frame, RouterError> {
8787    let body = serde_json::to_vec(reply).map_err(|err| {
8788        RouterError::backend(
8789            0,
8790            frame.header.corr,
8791            format!("failed to encode {label}: {err}"),
8792        )
8793    })?;
8794
8795    Frame::build_with_version(
8796        response_version(frame),
8797        FrameType::Response,
8798        control_flags(),
8799        0,
8800        0,
8801        frame.header.corr,
8802        body,
8803    )
8804    .map_err(RouterError::FrameBuild)
8805}
8806
8807/// Map a forwarding failure to the wire code a client sees.
8808///
8809/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
8810/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
8811/// chosen here decides whether a caller retries or gives up.
8812///
8813/// That makes attribution the load-bearing property, not merely having a code. A
8814/// permanent fault published as a retryable one produces a fleet-wide retry storm
8815/// against something that can never recover; a transient fault published as
8816/// permanent gives up on work that would have succeeded. Both look correct in a
8817/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
8818/// the mapping per variant rather than merely asserting that some code exists.
8819///
8820/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
8821/// two codes on the same side of the boundary passes it. Measured rather than
8822/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
8823/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
8824/// a test named for something else that happens to assert the string.
8825///
8826/// That accidental coverage is deliberately left alone rather than promoted to a
8827/// named test, because it guards a property this function does not promise.
8828/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
8829/// specific code within a class, so identity is free to change and only the
8830/// partition is a contract. Splitting it out would assert a guarantee nothing
8831/// depends on — and a suite that promises more than the code does is the harder
8832/// thing to correct later, because the next reader cannot tell which assertions
8833/// are load-bearing.
8834///
8835/// Pin identity here the moment a consumer branches on a specific code.
8836fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8837    match err {
8838        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8839        ForwardingError::NoModuleConnection => "target_unavailable",
8840        ForwardingError::ModuleReloading { .. } => "module_reloading",
8841        ForwardingError::ClientRouteChannelExhausted { .. }
8842        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8843        ForwardingError::StaleModuleEndpoint
8844        | ForwardingError::UnknownReservation { .. }
8845        | ForwardingError::ConnectionClosing { .. }
8846        | ForwardingError::ClientEgressClosed { .. }
8847        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8848        // Only a swap candidate's registration can produce this, and it means
8849        // exactly what a second active HELLO for a live id means.
8850        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8851        ForwardingError::RelayCorrelationExhausted
8852        | ForwardingError::RouteOpenBuild(_)
8853        | ForwardingError::Poisoned => "forwarding_error",
8854    }
8855}
8856
8857fn response_version(frame: &Frame) -> u8 {
8858    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8859        frame.header.ver
8860    } else {
8861        PROTOCOL_VERSION
8862    }
8863}
8864
8865fn control_flags() -> Flags {
8866    Flags::new(false, Priority::Passive, false)
8867}
8868
8869/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
8870/// channel. The target is the module (a client never saw the route), so this
8871/// takes the module path: delivered late rather than dropped when the module's
8872/// queue is momentarily full, and never closing its connection.
8873fn send_goodbye_target_best_effort(
8874    counters: &DaemonCounters,
8875    target: &GoodbyeTarget,
8876    context: &'static str,
8877) {
8878    let Ok(frame) = Frame::build_with_version(
8879        target.negotiated_ver,
8880        FrameType::Goodbye,
8881        control_flags(),
8882        target.channel,
8883        target.epoch,
8884        0,
8885        Vec::new(),
8886    ) else {
8887        return;
8888    };
8889    crate::forwarding::send_module_route_goodbye(
8890        counters,
8891        &target.sink,
8892        frame,
8893        target.module_id.as_deref(),
8894        context,
8895    );
8896}
8897
8898pub(crate) fn send_route_control_pushes(
8899    forwarding: &ForwardingTable,
8900    routes: Vec<EndpointRoute>,
8901    push: ClientControlPush,
8902) {
8903    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8904    for route in routes {
8905        let target = route.goodbye_target;
8906        if let Some((existing, channels)) = targets
8907            .iter_mut()
8908            .find(|(existing, _)| existing.connection_id == target.connection_id)
8909        {
8910            debug_assert_eq!(
8911                existing.negotiated_ver, target.negotiated_ver,
8912                "one connection cannot negotiate multiple frame versions"
8913            );
8914            if !channels.contains(&target.channel) {
8915                channels.push(target.channel);
8916            }
8917            continue;
8918        }
8919        let channel = target.channel;
8920        targets.push((target, vec![channel]));
8921    }
8922    for (target, mut channels) in targets {
8923        channels.sort_unstable();
8924        let mut push = push.clone();
8925        match &mut push {
8926            ClientControlPush::RouteClosing {
8927                channels: covered, ..
8928            }
8929            | ClientControlPush::RouteClosed {
8930                channels: covered, ..
8931            } => *covered = channels,
8932        }
8933        let body = match serde_json::to_vec(&push) {
8934            Ok(body) => body,
8935            Err(err) => {
8936                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8937                continue;
8938            }
8939        };
8940        let frame = match Frame::build_with_version(
8941            target.negotiated_ver,
8942            FrameType::Push,
8943            control_flags(),
8944            0,
8945            0,
8946            0,
8947            body.clone(),
8948        ) {
8949            Ok(frame) => frame,
8950            Err(err) => {
8951                warn!(
8952                    route_channel = target.channel,
8953                    error = %err,
8954                    "failed to build route lifecycle control PUSH frame"
8955                );
8956                continue;
8957            }
8958        };
8959        if let Err(err) = target.sink.try_send(frame) {
8960            if target.close_on_delivery_failure() {
8961                warn!(
8962                    target_connection_id = target.connection_id.get(),
8963                    route_channel = target.channel,
8964                    error = %err,
8965                    "route lifecycle control PUSH was not delivered to client; closing target connection"
8966                );
8967                let _ = forwarding.escalate_client_delivery_failure(
8968                    target.connection_id,
8969                    target.channel,
8970                    target.epoch,
8971                    CloseReason::new(
8972                        "route_lifecycle_push_delivery_failed",
8973                        format!(
8974                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8975                            target.channel
8976                        ),
8977                    ),
8978                    crate::forwarding::UndeliveredFrame {
8979                        module_id: target.module_id.as_deref(),
8980                        sink: &target.sink,
8981                    },
8982                );
8983            }
8984        }
8985    }
8986}
8987
8988#[cfg(test)]
8989mod tests {
8990    include!("control/agent_commands_tests.rs");
8991    include!("control/resource_revision_tests.rs");
8992    #[tokio::test]
8993    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8994        let registry = Arc::new(Registry::default());
8995        let handler = ControlHandler::new(registry);
8996        assert_eq!(
8997            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8998            None
8999        );
9000        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
9001        hello_via_sink(
9002            &handler,
9003            &ctx,
9004            &mut rx,
9005            hello_frame("daemon", PROTOCOL_VERSION, 1),
9006        )
9007        .await;
9008        assert_eq!(
9009            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
9010            Some("daemon".into())
9011        );
9012        assert_eq!(
9013            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
9014            None
9015        );
9016        assert_eq!(
9017            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
9018            None,
9019            "a registered module is not necessarily supervised"
9020        );
9021        assert_eq!(
9022            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
9023            None,
9024            "aggregate provenance has no single target"
9025        );
9026        assert_eq!(
9027            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
9028            None
9029        );
9030    }
9031
9032    mod peer_manifest {
9033        include!("peer_manifest_tests.rs");
9034    }
9035    mod activity_read {
9036        include!("activity_read_tests.rs");
9037    }
9038
9039    mod lifecycle_activity {
9040        include!("control_lifecycle_tests.rs");
9041    }
9042    #[cfg(unix)]
9043    #[tokio::test]
9044    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
9045        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
9046        let path = dir.join("subc.jsonc");
9047        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
9048            "program":"/bin/sleep","args":["60"],"protocol":"none",
9049            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
9050        }}}).to_string()).unwrap();
9051        let mut configured = crate::daemon_config::load(&path)
9052            .unwrap()
9053            .unwrap()
9054            .modules
9055            .pop()
9056            .unwrap();
9057        let registry = std::sync::Arc::new(crate::Registry::default());
9058        let handle = crate::SupervisorHandle::new();
9059        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
9060            .with_handle(handle.clone());
9061        let module = supervisor
9062            .supervise_configured_with_health(
9063                configured.module_spec(),
9064                true,
9065                configured.health.clone(),
9066                None,
9067                configured.restart,
9068            )
9069            .unwrap();
9070        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
9071        let before = module.status().unwrap().pid;
9072        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
9073        configured.health.cadence = std::time::Duration::from_secs(3600);
9074        let health_only = handler
9075            .reconcile_supervised_modules(
9076                &supervisor,
9077                vec![configured.clone()],
9078                false,
9079                Principal::Direct,
9080            )
9081            .await
9082            .unwrap();
9083        assert!(
9084            health_only.0.changed_pending_reload.is_empty(),
9085            "health policy is already applied live"
9086        );
9087        assert_eq!(module.status().unwrap().pid, before);
9088        assert_eq!(
9089            module.configuration().unwrap().1.http,
9090            configured.health.http
9091        );
9092        configured.args = vec!["61".into()];
9093        let launch = handler
9094            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
9095            .await
9096            .unwrap();
9097        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
9098        assert_eq!(
9099            module.status().unwrap().pid,
9100            before,
9101            "a launch edit is stored until reload"
9102        );
9103        module.drain().await.unwrap();
9104    }
9105    use cortexkit_test_support::ScratchDir;
9106    use std::{
9107        collections::BTreeMap,
9108        fmt,
9109        path::PathBuf,
9110        sync::{Arc, Mutex},
9111        time::Duration,
9112    };
9113
9114    use serde_json::{json, Value};
9115    use subc_protocol::{
9116        manifest::{
9117            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9118            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9119        },
9120        session::HealthStatus,
9121        FrameType,
9122    };
9123
9124    use super::*;
9125    use crate::{
9126        forwarding::{DataRoute, DataRouteState},
9127        registry::ChannelState,
9128        router::FrameSink,
9129        stderr_tail::DEFAULT_MAX_LINE_BYTES,
9130        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9131        RouteCtx, Router,
9132    };
9133    use tokio::{
9134        sync::mpsc,
9135        time::{sleep, Instant},
9136    };
9137    use tracing::{
9138        field::{Field, Visit},
9139        Event, Subscriber,
9140    };
9141    use tracing_subscriber::{layer::Context, prelude::*, Layer};
9142
9143    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
9144    ///
9145    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
9146    /// is only populated for `tests/*.rs` integration test binaries -- this file
9147    /// compiles as part of the library target, which gets neither. This test's
9148    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
9149    /// and the sibling binary lives two directories up at
9150    /// `<target-dir>/<profile>/fake-aft-stub`.
9151    ///
9152    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
9153    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
9154    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
9155    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
9156    /// `NotFound`, which reads as a broken test rather than an unbuilt
9157    /// dependency -- so state the cause and the remedy instead. Deliberately a
9158    /// panic and not a silent skip: a test that quietly passes when it could not
9159    /// run is worse than one that fails, because it reports health it never
9160    /// verified.
9161    fn fake_aft_stub_path() -> PathBuf {
9162        let mut path = std::env::current_exe().expect("current_exe available in tests");
9163        path.pop(); // .../deps/
9164        path.pop(); // .../<profile>/
9165        path.push(if cfg!(windows) {
9166            "fake-aft-stub.exe"
9167        } else {
9168            "fake-aft-stub"
9169        });
9170        assert!(
9171            path.exists(),
9172            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9173             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9174            path.display()
9175        );
9176        path
9177    }
9178
9179    /// Whether clients retry `code` in place: the predicate itself, never a copy
9180    /// of its set. A copied list breaks silently when a code is added to or
9181    /// removed from the real one, and a stale copy here would let exactly the
9182    /// failure this test exists to catch pass.
9183    fn client_retries(code: &str) -> bool {
9184        subc_protocol::error_codes::is_retryable_route_open(code)
9185    }
9186
9187    /// A code is not a label — clients branch on it, so publishing the wrong KIND
9188    /// of failure is worse than publishing none. A permanent fault dressed as
9189    /// retryable makes every client in the fleet retry forever against something
9190    /// that cannot recover; a transient fault dressed as permanent abandons work
9191    /// that would have succeeded.
9192    ///
9193    /// Asserting "a code exists" cannot catch either, because the string is free
9194    /// to say anything. This enumerates every variant and pins which side of the
9195    /// retry boundary it lands on, so a new variant must be classified here
9196    /// deliberately rather than inheriting whichever arm it was appended to.
9197    #[test]
9198    fn retryability_of_forwarding_codes_matches_the_failure() {
9199        // Transient by nature: the target is booting, reloading, or its endpoint
9200        // was swapped mid-flight. Retrying is how these resolve.
9201        let transient = [
9202            ForwardingError::NoModuleConnection,
9203            ForwardingError::ModuleReloading {
9204                module_id: "m".into(),
9205            },
9206            ForwardingError::StaleModuleEndpoint,
9207            ForwardingError::UnknownReservation {
9208                client_channel: 1,
9209                module_channel: 1,
9210            },
9211            ForwardingError::ConnectionClosing {
9212                connection_id: ConnectionId::new(1),
9213            },
9214            ForwardingError::ClientEgressClosed {
9215                connection_id: ConnectionId::new(1),
9216            },
9217            ForwardingError::ModuleEgressUnavailable {
9218                connection_id: ConnectionId::new(1),
9219            },
9220        ];
9221        for err in transient {
9222            let code = forwarding_error_code(&err);
9223            assert!(
9224                client_retries(code),
9225                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9226            );
9227        }
9228
9229        // Not fixed by retrying. Channel and correlation exhaustion need the
9230        // caller to close routes, and a poisoned lock is a daemon that cannot
9231        // recover at all — the worst thing to advertise as retryable, since every
9232        // client would storm a daemon that will never answer.
9233        let permanent = [
9234            ForwardingError::ConnectionRoleConflict {
9235                connection_id: ConnectionId::new(1),
9236            },
9237            ForwardingError::ClientRouteChannelExhausted {
9238                connection_id: ConnectionId::new(1),
9239            },
9240            ForwardingError::ModuleRouteChannelExhausted {
9241                endpoint: ModuleEndpointId {
9242                    connection_id: ConnectionId::new(1),
9243                    generation: 1,
9244                },
9245            },
9246            ForwardingError::RelayCorrelationExhausted,
9247            ForwardingError::RouteOpenBuild("x".into()),
9248            ForwardingError::Poisoned,
9249        ];
9250        for err in permanent {
9251            let code = forwarding_error_code(&err);
9252            assert!(
9253                !client_retries(code),
9254                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9255            );
9256        }
9257    }
9258
9259    /// The principal is the daemon's answer to "who is calling", and modules
9260    /// branch on it: aft gates bash on it, cerebellum gates browser control,
9261    /// plexus gates connector invocation. So a stamp is an authorization input in
9262    /// another process, not a label — and both possible answers SUCCEED, which is
9263    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
9264    /// first-party capability to something that never proved it; a supervised one
9265    /// stamped `Direct` silently strips a module of capability it is entitled to.
9266    ///
9267    /// Neither shows up in a test that only checks the bind succeeded. Before this
9268    /// test the only coverage was accidental —
9269    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
9270    /// stamped principal on its way past, so narrowing that wire-shape test to its
9271    /// stated subject would have deleted the last assertion on this value. It
9272    /// still asserts the stamp, which is now redundancy rather than the only
9273    /// guard: both fail under the same mutation, and this one names the reason.
9274    /// SCOPE: this handler's supervisor has spawned nothing, so
9275    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
9276    /// is unreachable here. Both assertions below are refusals, and a mutant that
9277    /// refuses everything would satisfy them.
9278    ///
9279    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
9280    /// spawns a supervised consumer, reads its live nonce, and asserts the module
9281    /// observed `principal.kind == "reserved"` carrying that module_id — verified
9282    /// at source rather than assumed, since a citation is a claim about another
9283    /// file and ages like one. Recorded because a harness that structurally
9284    /// cannot reach an arm reports "none" for that arm identically to one that
9285    /// covers it and found nothing.
9286    #[tokio::test]
9287    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9288        let handler = ControlHandler::default();
9289        let frame =
9290            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9291
9292        // Absent consumer_identity is the ordinary case: a human at a terminal, or
9293        // any process holding the connection file. Nothing was proved, so nothing
9294        // may be granted beyond the unattested floor.
9295        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9296        assert_eq!(
9297            stamped,
9298            Principal::Direct,
9299            "a caller that proved nothing must not be stamped as a supervised module"
9300        );
9301
9302        // A claimed module_id with a nonce no supervised child was given is a
9303        // forgery attempt, not a weaker caller: it must be REFUSED rather than
9304        // quietly demoted to Direct, or an impersonation attempt looks identical
9305        // to an ordinary unattested connection.
9306        let forged = handler
9307            .route_open_principal(
9308                &frame,
9309                Some(ConsumerIdentity {
9310                    module_id: "aft".to_string(),
9311                    launch_nonce: "not-a-real-nonce".to_string(),
9312                }),
9313            )
9314            .unwrap();
9315        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9316        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9317    }
9318
9319    /// The test above hands `route_open_principal` an identity it built itself,
9320    /// which proves the stamping rule and nothing about where the identity comes
9321    /// from. The real producer is a wire body, and the two are joined by a serde
9322    /// field name that nothing else asserts.
9323    ///
9324    /// That join fails quietly in one specific way: an unrecognised key is simply
9325    /// absent after parsing, so a renamed or misspelled `consumer_identity`
9326    /// yields `None` and every supervised module silently drops to `Direct`.
9327    /// Capability-wise that is the safe direction, but it surfaces far from its
9328    /// cause — as a module mysteriously refused bash — and it would pass every
9329    /// test that builds its own input.
9330    ///
9331    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
9332    /// would break every client the moment the daemon gains a field, trading a
9333    /// quiet demotion for a hard refusal on additive change. Asserting the join
9334    /// instead means a rename breaks a test here rather than the fleet.
9335    #[test]
9336    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9337        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9338        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9339        let ClientControlRequest::RouteOpen {
9340            consumer_identity, ..
9341        } = parsed
9342        else {
9343            panic!("route.open body must parse as RouteOpen");
9344        };
9345        assert_eq!(
9346            consumer_identity,
9347            Some(ConsumerIdentity {
9348                module_id: "aft".to_string(),
9349                launch_nonce: "n".to_string(),
9350            }),
9351            "the wire field name must reach the value route_open_principal reads"
9352        );
9353    }
9354
9355    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9356        ModuleManifest::builder(module_id, "0.1.0")
9357            .protocol_ver(protocol_ver)
9358            .provides(vec![ProviderRole::ToolProvider {
9359                tools: vec![Tool {
9360                    name: "read".to_string(),
9361                    description: None,
9362                    execution_mode: ExecutionMode::Pure,
9363                    schema: json!({"type": "object"}),
9364                }],
9365                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9366                concurrency: Concurrency::ModuleManaged,
9367                emits_push: true,
9368                sub_supervises: true,
9369            }])
9370            .build()
9371    }
9372
9373    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9374        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9375    }
9376
9377    fn hello_frame_with_control_ops(
9378        module_id: &str,
9379        protocol_ver: u8,
9380        corr: u64,
9381        control_ops: Option<Vec<String>>,
9382    ) -> Frame {
9383        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9384    }
9385
9386    fn hello_frame_with_nonce(
9387        module_id: &str,
9388        protocol_ver: u8,
9389        corr: u64,
9390        launch_nonce: Option<&str>,
9391    ) -> Frame {
9392        hello_frame_full(
9393            module_id,
9394            protocol_ver,
9395            corr,
9396            None,
9397            launch_nonce.map(ToOwned::to_owned),
9398        )
9399    }
9400
9401    fn hello_frame_full(
9402        module_id: &str,
9403        protocol_ver: u8,
9404        corr: u64,
9405        control_ops: Option<Vec<String>>,
9406        launch_nonce: Option<String>,
9407    ) -> Frame {
9408        let body = serde_json::to_vec(&ModuleHelloBody {
9409            manifest: manifest(module_id, protocol_ver),
9410            protocol_ver,
9411            control_ops,
9412            launch_nonce,
9413        })
9414        .unwrap();
9415        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9416    }
9417
9418    fn hello_frame_with_rich_ui(
9419        module_id: &str,
9420        protocol_ver: u8,
9421        corr: u64,
9422        launch_nonce: Option<&str>,
9423        rich_ui: Value,
9424    ) -> Frame {
9425        let mut body = serde_json::to_value(ModuleHelloBody {
9426            manifest: manifest(module_id, protocol_ver),
9427            protocol_ver,
9428            control_ops: None,
9429            launch_nonce: launch_nonce.map(str::to_string),
9430        })
9431        .unwrap();
9432        body["manifest"]["rich_ui"] = rich_ui;
9433        Frame::build(
9434            FrameType::Hello,
9435            control_flags(),
9436            0,
9437            0,
9438            corr,
9439            serde_json::to_vec(&body).unwrap(),
9440        )
9441        .unwrap()
9442    }
9443
9444    fn non_routable_hello_frame_with_control_ops(
9445        module_id: &str,
9446        corr: u64,
9447        control_ops: Option<Vec<String>>,
9448    ) -> Frame {
9449        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9450        manifest.provides.clear();
9451        let body = serde_json::to_vec(&ModuleHelloBody {
9452            manifest,
9453            protocol_ver: PROTOCOL_VERSION,
9454            control_ops,
9455            launch_nonce: None,
9456        })
9457        .unwrap();
9458        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9459    }
9460
9461    fn capability_grammar_hello_frame(
9462        capabilities: Value,
9463        runtime_computed: Option<Value>,
9464        corr: u64,
9465    ) -> Frame {
9466        let mut body = serde_json::to_value(ModuleHelloBody {
9467            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9468            protocol_ver: PROTOCOL_VERSION,
9469            control_ops: None,
9470            launch_nonce: None,
9471        })
9472        .expect("HELLO body serializes");
9473        body["manifest"]["capabilities"] = capabilities;
9474        if let Some(runtime_computed) = runtime_computed {
9475            body["runtime_computed"] = runtime_computed;
9476        }
9477        Frame::build(
9478            FrameType::Hello,
9479            control_flags(),
9480            0,
9481            0,
9482            corr,
9483            serde_json::to_vec(&body).expect("HELLO body reserializes"),
9484        )
9485        .expect("HELLO frame builds")
9486    }
9487
9488    fn channel_request(channel: u16, corr: u64) -> Frame {
9489        Frame::build(
9490            FrameType::Request,
9491            Flags::new(true, Priority::Interactive, false),
9492            channel,
9493            0,
9494            corr,
9495            b"opaque".to_vec(),
9496        )
9497        .unwrap()
9498    }
9499
9500    fn route_ctx(
9501        connection_id: ConnectionId,
9502    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9503        let (tx, rx) = mpsc::channel(8);
9504        (
9505            RouteCtx {
9506                connection_id,
9507                egress: FrameSink::new(tx),
9508            },
9509            rx,
9510        )
9511    }
9512
9513    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9514        serde_json::from_slice(&frame.body).unwrap()
9515    }
9516
9517    /// Register a module over a connection that has a sink and return the
9518    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
9519    /// module's own sink rather than returning it as a reply, so the ack is
9520    /// taken off `rx` here and whatever the test reads next is what followed it.
9521    async fn hello_via_sink(
9522        handler: &ControlHandler,
9523        ctx: &RouteCtx,
9524        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9525        hello: Frame,
9526    ) -> Frame {
9527        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9528        assert!(
9529            replies.is_empty(),
9530            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9531        );
9532        let ack = rx
9533            .try_recv()
9534            .expect("HELLO_ACK is queued on the module sink")
9535            .frame;
9536        assert_eq!(ack.header.ty, FrameType::HelloAck);
9537        ack
9538    }
9539
9540    fn parse_error(frame: &Frame) -> Value {
9541        serde_json::from_slice(&frame.body).unwrap()
9542    }
9543
9544    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9545        serde_json::from_slice(&frame.body).unwrap()
9546    }
9547
9548    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9549        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9550            route_channel,
9551            route_epoch: 0,
9552            kind,
9553        })
9554        .unwrap();
9555        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9556    }
9557
9558    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9559        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9560            module_id: module_id.to_string(),
9561        })
9562        .unwrap();
9563        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9564    }
9565
9566    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9567        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9568    }
9569
9570    // Exercise retired opener fields without putting them back in BindIdentity.
9571    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9572        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9573        let mut frame = route_open_frame(corr, module_id, root);
9574        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9575        body["identity"]["project_root"] = serde_json::json!(path.as_path());
9576        frame.body = serde_json::to_vec(&body).unwrap();
9577        frame
9578    }
9579
9580    fn route_open_frame_with_consumer_capabilities(
9581        corr: u64,
9582        module_id: &str,
9583        _project_root: ScratchDir,
9584        consumer_capabilities: Option<Vec<String>>,
9585    ) -> Frame {
9586        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9587            busy_refusals: None,
9588            target: RouteTarget::ToolProvider {
9589                module_id: module_id.to_string(),
9590            },
9591            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9592            consumer_identity: None,
9593            consumer_capabilities,
9594            role_versions: None,
9595            admission_facts: None,
9596            scope: None,
9597        })
9598        .unwrap();
9599        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9600    }
9601
9602    fn route_open_frame_with_role_versions(
9603        corr: u64,
9604        module_id: &str,
9605        _project_root: ScratchDir,
9606        role_versions: Option<BTreeMap<String, String>>,
9607    ) -> Frame {
9608        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9609            busy_refusals: None,
9610            target: RouteTarget::ToolProvider {
9611                module_id: module_id.to_string(),
9612            },
9613            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9614            consumer_identity: None,
9615            consumer_capabilities: None,
9616            role_versions,
9617            admission_facts: None,
9618            scope: None,
9619        })
9620        .unwrap();
9621        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9622    }
9623
9624    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9625        entries
9626            .iter()
9627            .map(|(role, version)| (role.to_string(), version.to_string()))
9628            .collect()
9629    }
9630
9631    fn route_open_frame_with_admission_facts(
9632        corr: u64,
9633        module_id: &str,
9634        _project_root: ScratchDir,
9635        consumer_identity: Option<subc_control::ConsumerIdentity>,
9636        facts: Option<Value>,
9637    ) -> Frame {
9638        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9639            busy_refusals: None,
9640            target: RouteTarget::ToolProvider {
9641                module_id: module_id.to_string(),
9642            },
9643            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9644            consumer_identity,
9645            consumer_capabilities: None,
9646            role_versions: None,
9647            admission_facts: facts,
9648            scope: None,
9649        })
9650        .unwrap();
9651        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9652    }
9653
9654    #[derive(Clone, Default)]
9655    struct EventCapture {
9656        events: Arc<Mutex<Vec<CapturedEvent>>>,
9657    }
9658
9659    #[derive(Clone, Debug)]
9660    struct CapturedEvent {
9661        target: String,
9662        level: tracing::Level,
9663        fields: BTreeMap<String, String>,
9664    }
9665
9666    impl EventCapture {
9667        fn events(&self) -> Vec<CapturedEvent> {
9668            self.events.lock().unwrap().clone()
9669        }
9670    }
9671
9672    impl<S> Layer<S> for EventCapture
9673    where
9674        S: Subscriber,
9675    {
9676        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9677            let mut visitor = EventFieldVisitor::default();
9678            event.record(&mut visitor);
9679            self.events.lock().unwrap().push(CapturedEvent {
9680                target: event.metadata().target().to_string(),
9681                level: *event.metadata().level(),
9682                fields: visitor.fields,
9683            });
9684        }
9685    }
9686
9687    #[derive(Default)]
9688    struct EventFieldVisitor {
9689        fields: BTreeMap<String, String>,
9690    }
9691
9692    impl Visit for EventFieldVisitor {
9693        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9694            self.fields
9695                .insert(field.name().to_string(), format!("{value:?}"));
9696        }
9697    }
9698
9699    fn health_response(corr: u64, status: HealthStatus) -> Frame {
9700        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9701            status,
9702            detail: Some("warming".to_string()),
9703            metrics: Some(json!({"queue_depth": 3})),
9704        })
9705        .unwrap();
9706        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9707    }
9708
9709    fn route_bind_ack(corr: u64) -> Frame {
9710        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9711        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9712    }
9713
9714    fn unique_project_root(label: &str) -> ScratchDir {
9715        ScratchDir::new(label)
9716    }
9717
9718    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9719        match parse_route_poll(frame) {
9720            ClientControlResponse::RoutePoll {
9721                status: None,
9722                live: Some(live),
9723                ..
9724            } => assert_eq!(live, expected_live),
9725            other => panic!("unexpected route.poll response: {other:?}"),
9726        }
9727    }
9728
9729    fn bind_liveness_route(
9730        registry: &Registry,
9731        forwarding: &ForwardingTable,
9732        module_id: &str,
9733    ) -> (RouteCtx, u16, u32) {
9734        let module_connection = ConnectionId::new(101);
9735        let client_connection = ConnectionId::new(202);
9736        let registration = registry
9737            .register_with_control_ops(
9738                manifest(module_id, PROTOCOL_VERSION),
9739                PROTOCOL_VERSION,
9740                module_connection,
9741                module_baseline_control_ops(),
9742            )
9743            .unwrap();
9744        let (module_tx, _module_rx) = mpsc::channel(8);
9745        let endpoint = forwarding
9746            .register_module_connection(
9747                module_connection,
9748                module_id.to_string(),
9749                PROTOCOL_VERSION,
9750                manifest_concurrency(&registration.manifest),
9751                FrameSink::new(module_tx),
9752            )
9753            .unwrap();
9754        let (client_ctx, _client_rx) = route_ctx(client_connection);
9755        let pending = forwarding
9756            .begin_route_bind_relay_for_test(
9757                client_connection,
9758                client_ctx.egress.clone(),
9759                1,
9760                module_id,
9761            )
9762            .unwrap();
9763        assert_eq!(pending.endpoint, endpoint);
9764        let route_channel = pending.client_channel;
9765        let route_epoch = pending.client_epoch;
9766        forwarding
9767            .complete_pending_relay(
9768                module_connection,
9769                pending.corr,
9770                RouteBindRelayOutcome::Accepted,
9771            )
9772            .unwrap();
9773        (client_ctx, route_channel, route_epoch)
9774    }
9775
9776    struct FakeProcessLiveness {
9777        live: Option<bool>,
9778    }
9779
9780    impl ModuleProcessLiveness for FakeProcessLiveness {
9781        fn process_live(&self, _module_id: &str) -> Option<bool> {
9782            self.live
9783        }
9784    }
9785
9786    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9787    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9788    {
9789        let registry = Arc::new(Registry::default());
9790        let supervisor_handle = SupervisorHandle::new();
9791        let supervisor = Supervisor::new_for_test(
9792            Arc::clone(&registry),
9793            RestartPolicy::new(1, Duration::from_millis(10)),
9794        )
9795        .with_handle(supervisor_handle.clone());
9796        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9797        let module = supervisor
9798            .spawn(ModuleSpec {
9799                carry_containment: false,
9800                module_id: "stderr-tail-wire".to_string(),
9801                program: fake_aft_stub_path(),
9802                args: Vec::new(),
9803                env: vec![
9804                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9805                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9806                ],
9807                reserved: false,
9808                reserved_prefixes: Vec::new(),
9809                protocol: ModuleProtocol::Subc,
9810                overlap: Default::default(),
9811            })
9812            .unwrap();
9813
9814        let deadline = Instant::now() + Duration::from_secs(5);
9815        loop {
9816            let tail = module.stderr_tail(None, None);
9817            if tail
9818                .entries
9819                .iter()
9820                .any(|entry| matches!(entry, TailEntry::ProcessStart))
9821                && tail.entries.iter().any(|entry| {
9822                    matches!(
9823                        entry,
9824                        TailEntry::Line {
9825                            truncated: true,
9826                            ..
9827                        }
9828                    )
9829                })
9830            {
9831                break;
9832            }
9833            assert!(
9834                Instant::now() < deadline,
9835                "module did not produce a truncated line and restart boundary: {tail:?}"
9836            );
9837            sleep(Duration::from_millis(10)).await;
9838        }
9839
9840        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9841        let request = ClientControlRequest::SupervisorStderrTail {
9842            module_id: "stderr-tail-wire".to_string(),
9843            max_lines: None,
9844            max_bytes: None,
9845        };
9846        let frame = Frame::build(
9847            FrameType::Request,
9848            control_flags(),
9849            0,
9850            0,
9851            1,
9852            serde_json::to_vec(&request).unwrap(),
9853        )
9854        .unwrap();
9855        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9856        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9857        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9858            serde_json::from_slice(&responses[0].body).unwrap()
9859        else {
9860            panic!("expected supervisor.stderr_tail response");
9861        };
9862
9863        assert!(
9864            tail.entries
9865                .iter()
9866                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9867            "the control response lost the restart boundary"
9868        );
9869        let Some(StderrTailEntry::Line {
9870            text,
9871            truncated,
9872            at_ms,
9873        }) = tail.entries.iter().find(|entry| {
9874            matches!(
9875                entry,
9876                StderrTailEntry::Line {
9877                    truncated: true,
9878                    ..
9879                }
9880            )
9881        })
9882        else {
9883            panic!("the control response lost the truncated line");
9884        };
9885        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9886        assert!(*truncated);
9887        assert!(
9888            at_ms.is_some(),
9889            "the control response lost the line's capture time"
9890        );
9891    }
9892
9893    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
9894    /// read done on the worker thread would stall every other task until it
9895    /// finished; the read must run off the worker so this test's own task keeps
9896    /// running while the read is paused.
9897    #[tokio::test(flavor = "current_thread")]
9898    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9899        let dir = ScratchDir::new("terminals-off-worker");
9900        let journal_path = dir.join("terminals.jsonl");
9901        let registry = Arc::new(Registry::default());
9902        let supervisor_handle = SupervisorHandle::new();
9903        let supervisor =
9904            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9905                .with_handle(supervisor_handle.clone())
9906                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9907        let module = supervisor
9908            .spawn(ModuleSpec {
9909                carry_containment: false,
9910                module_id: "terminal-off-worker".to_string(),
9911                program: fake_aft_stub_path(),
9912                args: Vec::new(),
9913                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9914                reserved: false,
9915                reserved_prefixes: Vec::new(),
9916                protocol: ModuleProtocol::Subc,
9917                overlap: Default::default(),
9918            })
9919            .unwrap();
9920        let deadline = Instant::now() + Duration::from_secs(5);
9921        while module.terminal_history().entries.len() != 2 {
9922            assert!(Instant::now() < deadline, "module did not record two exits");
9923            sleep(Duration::from_millis(10)).await;
9924        }
9925
9926        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9927        let handler =
9928            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
9929        let frame = Frame::build(
9930            FrameType::Request,
9931            control_flags(),
9932            0,
9933            0,
9934            1,
9935            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9936                module_id: "terminal-off-worker".to_string(),
9937            })
9938            .unwrap(),
9939        )
9940        .unwrap();
9941        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9942        let spawned_at = std::time::Instant::now();
9943        let read = tokio::spawn({
9944            let handler = Arc::clone(&handler);
9945            async move { handler.handle_control_frame(&ctx, frame).await }
9946        });
9947        // Waiting for the pause from a blocking thread keeps this task pending,
9948        // so the runtime's single worker is free to run the read task.
9949        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9950            .await
9951            .unwrap()
9952            .expect("the history read reached its pause");
9953        let elapsed = spawned_at.elapsed();
9954        assert!(
9955            elapsed < Duration::from_secs(2) && !read.is_finished(),
9956            "this task could not run while the history read was paused \
9957             (resumed after {elapsed:?}, read finished: {})",
9958            read.is_finished()
9959        );
9960
9961        drop(release);
9962        let responses = read.await.unwrap().unwrap();
9963        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9964        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9965            panic!("expected supervisor.terminals response");
9966        };
9967        assert_eq!(terminals.entries.len(), 2);
9968        assert_eq!(terminals.journal_skipped_lines, 0);
9969        assert_eq!(terminals.journal_read_errors, 0);
9970    }
9971
9972    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9973    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9974        let registry = Arc::new(Registry::default());
9975        let supervisor_handle = SupervisorHandle::new();
9976        let supervisor =
9977            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
9978                .with_handle(supervisor_handle.clone());
9979        let module = supervisor
9980            .spawn(ModuleSpec {
9981                carry_containment: false,
9982                module_id: "terminal-golden".to_string(),
9983                program: fake_aft_stub_path(),
9984                args: Vec::new(),
9985                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9986                reserved: false,
9987                reserved_prefixes: Vec::new(),
9988                protocol: ModuleProtocol::Subc,
9989                overlap: Default::default(),
9990            })
9991            .unwrap();
9992
9993        let deadline = Instant::now() + Duration::from_secs(5);
9994        while module.terminal_history().entries.len() != 2 {
9995            assert!(
9996                Instant::now() < deadline,
9997                "module did not retain two terminal exits: {:?}",
9998                module.terminal_history()
9999            );
10000            sleep(Duration::from_millis(10)).await;
10001        }
10002
10003        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10004        let request = ClientControlRequest::SupervisorTerminals {
10005            module_id: "terminal-golden".to_string(),
10006        };
10007        let frame = Frame::build(
10008            FrameType::Request,
10009            control_flags(),
10010            0,
10011            0,
10012            1,
10013            serde_json::to_vec(&request).unwrap(),
10014        )
10015        .unwrap();
10016        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
10017        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
10018        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
10019        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
10020            panic!("expected supervisor.terminals response");
10021        };
10022        assert_eq!(terminals.entries.len(), 2);
10023        assert_eq!(terminals.dropped, 0);
10024
10025        let mut rendered = serde_json::to_value(response).unwrap();
10026        // Wall-clock fields are the observation contract, but not stable fixture
10027        // bytes; normalize only them after the real handler has shaped the response.
10028        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
10029        for (index, entry) in rendered["entries"]
10030            .as_array_mut()
10031            .expect("terminal response entries array")
10032            .iter_mut()
10033            .enumerate()
10034        {
10035            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
10036        }
10037
10038        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10039            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
10040        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
10041        if std::env::var_os("UPDATE_GOLDEN").is_some() {
10042            std::fs::write(&golden_path, &serialized).unwrap();
10043        }
10044        let expected: Value =
10045            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
10046        assert_eq!(rendered, expected);
10047    }
10048
10049    #[test]
10050    fn hello_registers_manifest_and_returns_ack() {
10051        let registry = Arc::new(Registry::default());
10052        let handler = ControlHandler::new(Arc::clone(&registry));
10053        let conn = ConnectionId::new(1);
10054
10055        let responses = handler
10056            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
10057            .unwrap();
10058
10059        assert_eq!(responses.len(), 1);
10060        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10061        assert_eq!(responses[0].header.channel, 0);
10062        assert_eq!(responses[0].header.corr, 7);
10063        let ack = parse_ack(&responses[0]);
10064        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10065        assert!(ack
10066            .subc_capabilities
10067            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
10068        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
10069        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
10070        assert!(ack
10071            .subc_ops
10072            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
10073        assert!(ack
10074            .subc_ops
10075            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
10076
10077        let registration = registry.get_module("aft").unwrap().unwrap();
10078        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
10079        assert_eq!(registration.state, ChannelState::Active);
10080        assert_eq!(registration.connection_id, conn);
10081        assert_eq!(registration.control_ops, module_baseline_control_ops());
10082        assert!(registration.manifest.display_name.is_none());
10083    }
10084
10085    #[test]
10086    fn hello_display_name_refuses_invalid_manifest_by_field() {
10087        let registry = Arc::new(Registry::default());
10088        let handler = ControlHandler::new(Arc::clone(&registry));
10089        for (index, invalid) in [
10090            json!(""),
10091            json!("a".repeat(41)),
10092            json!(" Routing"),
10093            json!("a\nb"),
10094            json!("a\u{202e}b"),
10095            json!(7),
10096        ]
10097        .into_iter()
10098        .enumerate()
10099        {
10100            let module_id = format!("bad-display-name-{index}");
10101            let mut body = serde_json::to_value(ModuleHelloBody {
10102                manifest: manifest(&module_id, PROTOCOL_VERSION),
10103                protocol_ver: PROTOCOL_VERSION,
10104                control_ops: None,
10105                launch_nonce: None,
10106            })
10107            .unwrap();
10108            body["manifest"]["display_name"] = invalid;
10109            let hello = Frame::build(
10110                FrameType::Hello,
10111                control_flags(),
10112                0,
10113                0,
10114                7,
10115                serde_json::to_vec(&body).unwrap(),
10116            )
10117            .unwrap();
10118            let response = handler
10119                .handle_control(ConnectionId::new(100 + index as u64), hello)
10120                .unwrap();
10121            assert_eq!(response[0].header.ty, FrameType::Error);
10122            let error = parse_error(&response[0]);
10123            assert_eq!(error["code"], "invalid_manifest");
10124            assert!(error["message"].as_str().unwrap().contains("display_name"));
10125            assert!(registry.get_module(&module_id).unwrap().is_none());
10126        }
10127    }
10128
10129    #[test]
10130    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10131        let invalid_identifiers = [
10132            ("case_change", "credentials-Provider/v1"),
10133            ("leading_zero", "credentials-provider/v01"),
10134            ("trailing_hyphen", "credentials-provider-/v1"),
10135            ("consecutive_hyphens", "credentials--provider/v1"),
10136            ("uppercase", "Credentials-provider/v1"),
10137            ("missing_v", "credentials-provider/1"),
10138            ("whitespace", "credentials provider/v1"),
10139            ("zero_version", "credentials-provider/v0"),
10140            ("out_of_range_version", "credentials-provider/v4294967296"),
10141            (
10142                "overlength_name",
10143                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10144            ),
10145        ];
10146        let mut cases = invalid_identifiers
10147            .into_iter()
10148            .map(|(name, identifier)| {
10149                (
10150                    format!("identifier_{name}"),
10151                    "capabilities.provides[0]".to_string(),
10152                    identifier.to_string(),
10153                    json!({ "provides": [identifier] }),
10154                    None,
10155                )
10156            })
10157            .collect::<Vec<_>>();
10158        cases.extend([
10159            (
10160                "unknown_need".to_string(),
10161                "capabilities.requires[0].need".to_string(),
10162                "deferred".to_string(),
10163                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10164                None,
10165            ),
10166            (
10167                "duplicate_provides".to_string(),
10168                "capabilities.provides[1]".to_string(),
10169                "credentials-provider/v1".to_string(),
10170                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10171                None,
10172            ),
10173            (
10174                "duplicate_must_never_reach".to_string(),
10175                "capabilities.must_never_reach[1]".to_string(),
10176                "credentials-provider/v1".to_string(),
10177                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10178                None,
10179            ),
10180            (
10181                "duplicate_requires_same_need".to_string(),
10182                "capabilities.requires[1]".to_string(),
10183                "credentials-provider/v1".to_string(),
10184                json!({ "requires": [
10185                    { "capability": "credentials-provider/v1", "need": "required" },
10186                    { "capability": "credentials-provider/v1", "need": "required" }
10187                ] }),
10188                None,
10189            ),
10190            (
10191                "duplicate_requires_conflicting_need".to_string(),
10192                "capabilities.requires[1]".to_string(),
10193                "credentials-provider/v1".to_string(),
10194                json!({ "requires": [
10195                    { "capability": "credentials-provider/v1", "need": "required" },
10196                    { "capability": "credentials-provider/v1", "need": "optional" }
10197                ] }),
10198                None,
10199            ),
10200            (
10201                "capabilities_root_pointer".to_string(),
10202                "runtime_computed[0]".to_string(),
10203                "/capabilities".to_string(),
10204                json!({}),
10205                Some(json!(["/capabilities"])),
10206            ),
10207            (
10208                "capabilities_descendant_pointer".to_string(),
10209                "runtime_computed[0]".to_string(),
10210                "/capabilities/provides".to_string(),
10211                json!({}),
10212                Some(json!(["/capabilities/provides"])),
10213            ),
10214            (
10215                "malformed_pointer_without_leading_slash".to_string(),
10216                "runtime_computed[0]".to_string(),
10217                "capabilities".to_string(),
10218                json!({}),
10219                Some(json!(["capabilities"])),
10220            ),
10221            (
10222                "malformed_pointer_escape".to_string(),
10223                "runtime_computed[0]".to_string(),
10224                "/roles/~2/tools".to_string(),
10225                json!({}),
10226                Some(json!(["/roles/~2/tools"])),
10227            ),
10228            (
10229                "unknown_capabilities_field".to_string(),
10230                "capabilities.future".to_string(),
10231                "<array>".to_string(),
10232                json!({ "future": [] }),
10233                None,
10234            ),
10235        ]);
10236
10237        for (index, (name, field, value, capabilities, runtime_computed)) in
10238            cases.into_iter().enumerate()
10239        {
10240            let registry = Arc::new(Registry::default());
10241            let handler = ControlHandler::new(Arc::clone(&registry));
10242            let response = handler
10243                .handle_control(
10244                    ConnectionId::new((index + 1) as u64),
10245                    capability_grammar_hello_frame(
10246                        capabilities,
10247                        runtime_computed,
10248                        index as u64 + 1,
10249                    ),
10250                )
10251                .expect("invalid HELLO returns a refusal");
10252
10253            assert_eq!(response.len(), 1, "{name} must emit one refusal");
10254            let error = parse_error(&response[0]);
10255            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10256            let message = error["message"]
10257                .as_str()
10258                .expect("error message is a string");
10259            assert!(
10260                message.contains(&field),
10261                "{name}: field missing from {message}"
10262            );
10263            assert!(
10264                message.contains(&value),
10265                "{name}: value missing from {message}"
10266            );
10267            assert_eq!(
10268                registry
10269                    .active_registration_count()
10270                    .expect("registry reads"),
10271                0,
10272                "{name}: refused HELLO must not create a catalog entry"
10273            );
10274        }
10275    }
10276
10277    #[test]
10278    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10279        let registry = Arc::new(Registry::default());
10280        let handler = ControlHandler::new(Arc::clone(&registry));
10281        let capabilities = json!({
10282            "provides": ["credentials-provider/v1"],
10283            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10284            "must_never_reach": ["federation-transport/v1"]
10285        });
10286        let response = handler
10287            .handle_control(
10288                ConnectionId::new(99),
10289                capability_grammar_hello_frame(
10290                    capabilities.clone(),
10291                    Some(json!(["/roles/0/tools"])),
10292                    99,
10293                ),
10294            )
10295            .expect("valid HELLO registers");
10296        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10297
10298        let request = Frame::build(
10299            FrameType::Request,
10300            control_flags(),
10301            0,
10302            0,
10303            100,
10304            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10305                .expect("catalog request serializes"),
10306        )
10307        .expect("catalog request frame builds");
10308        let response = handler
10309            .handle_catalog_list(request, None)
10310            .expect("catalog list succeeds");
10311        let ClientControlResponse::CatalogList { modules, .. } =
10312            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10313        else {
10314            panic!("catalog request must return catalog.list");
10315        };
10316        assert_eq!(modules.len(), 1);
10317        assert_eq!(
10318            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10319            capabilities
10320        );
10321    }
10322
10323    #[test]
10324    fn catalog_list_mirrors_management_operation_description() {
10325        let registry = Arc::new(Registry::default());
10326        let handler = ControlHandler::new(Arc::clone(&registry));
10327        let description = "List managed records and return their identifiers and metadata.";
10328        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10329        manifest.provides = vec![ProviderRole::ManagementSurface {
10330            operations: vec![ManagementOperation {
10331                name: "records.list".to_string(),
10332                kind: ManagementOperationKind::Query,
10333                description: Some(description.to_string()),
10334            }],
10335            config_schema: json!({"type": "object"}),
10336            observability: vec![ObservabilitySurface {
10337                name: "records.stats".to_string(),
10338                kind: ObservabilityKind::Snapshot,
10339            }],
10340            identity_scope: vec![IdentityScope::Project],
10341            concurrency: Concurrency::ModuleManaged,
10342        }];
10343        registry
10344            .register_with_control_ops(
10345                manifest,
10346                PROTOCOL_VERSION,
10347                ConnectionId::new(99),
10348                Vec::new(),
10349            )
10350            .expect("described management manifest registers");
10351
10352        let request = Frame::build(
10353            FrameType::Request,
10354            control_flags(),
10355            0,
10356            0,
10357            100,
10358            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10359                .expect("catalog request serializes"),
10360        )
10361        .expect("catalog request frame builds");
10362        let response = handler
10363            .handle_catalog_list(request, None)
10364            .expect("catalog list succeeds");
10365        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10366        assert_eq!(
10367            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10368            "catalog.list must preserve the declared operation description verbatim"
10369        );
10370    }
10371
10372    #[test]
10373    fn catalog_list_mirrors_registered_operation_delivery() {
10374        let registry = Arc::new(Registry::default());
10375        let handler = ControlHandler::new(Arc::clone(&registry));
10376        let delivery =
10377            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10378        let mut body = serde_json::to_value(ModuleHelloBody {
10379            manifest: manifest("watch-management", PROTOCOL_VERSION),
10380            protocol_ver: PROTOCOL_VERSION,
10381            control_ops: None,
10382            launch_nonce: None,
10383        })
10384        .unwrap();
10385        body["manifest"]["provides"] = json!([{
10386            "role": "management_surface",
10387            "operations": [
10388                {"name": "status", "kind": "query"},
10389                {"name": "future", "kind": "query"},
10390                {"name": "reset", "kind": "mutate"}
10391            ],
10392            "config_schema": {}, "observability": [], "identity_scope": []
10393        }]);
10394        body["manifest"]["operation_delivery"] = delivery.clone();
10395        let hello = Frame::build(
10396            FrameType::Hello,
10397            control_flags(),
10398            0,
10399            0,
10400            99,
10401            serde_json::to_vec(&body).unwrap(),
10402        )
10403        .unwrap();
10404        let response = handler
10405            .handle_control(ConnectionId::new(99), hello)
10406            .unwrap();
10407        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10408
10409        let (_, registrations) = registry.list_modules().unwrap();
10410        let registered = &registrations[0].manifest;
10411        assert!(registered.is_watch_operation("status"));
10412        assert!(!registered.is_watch_operation("future"));
10413        assert!(!registered.is_watch_operation("reset"));
10414        assert!(!registered.is_watch_operation("missing"));
10415
10416        let request = Frame::build(
10417            FrameType::Request,
10418            control_flags(),
10419            0,
10420            0,
10421            100,
10422            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10423        )
10424        .unwrap();
10425        let response = handler.handle_catalog_list(request, None).unwrap();
10426        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10427        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10428        let ClientControlResponse::CatalogList { modules, .. } =
10429            serde_json::from_slice(&response[0].body).unwrap()
10430        else {
10431            panic!("expected catalog.list");
10432        };
10433        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10434    }
10435
10436    #[test]
10437    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10438        let registry = Arc::new(Registry::default());
10439        let supervisor = SupervisorHandle::new();
10440        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10441        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10442        let descriptor = json!({
10443            "schema": "ckios.richui.manifest/1",
10444            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10445            "bytes": 4096,
10446            "chunks": 4
10447        });
10448        let connection = ConnectionId::new(201);
10449        let hello = handler
10450            .handle_control(
10451                connection,
10452                hello_frame_with_rich_ui(
10453                    "vault",
10454                    PROTOCOL_VERSION,
10455                    201,
10456                    Some("the-real-nonce"),
10457                    descriptor.clone(),
10458                ),
10459            )
10460            .unwrap();
10461        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10462
10463        let request =
10464            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10465        let provides = registry
10466            .get_module_by_connection(connection)
10467            .unwrap()
10468            .unwrap()
10469            .manifest
10470            .provides;
10471        handler
10472            .handle_catalog_update(connection, request, provides, None, Some(false))
10473            .unwrap();
10474
10475        let request = Frame::build(
10476            FrameType::Request,
10477            control_flags(),
10478            0,
10479            0,
10480            203,
10481            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10482        )
10483        .unwrap();
10484        let response = handler.handle_catalog_list(request, None).unwrap();
10485        let ClientControlResponse::CatalogList { modules, .. } =
10486            serde_json::from_slice(&response[0].body).unwrap()
10487        else {
10488            panic!("expected catalog.list");
10489        };
10490        assert_eq!(
10491            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10492            descriptor
10493        );
10494        assert!(modules[0].nonce_verified);
10495    }
10496
10497    fn hello_with_role_declarations(declarations: Value) -> Frame {
10498        let mut body = serde_json::to_value(ModuleHelloBody {
10499            manifest: manifest("role-test", PROTOCOL_VERSION),
10500            protocol_ver: PROTOCOL_VERSION,
10501            control_ops: None,
10502            launch_nonce: None,
10503        })
10504        .unwrap();
10505        body["manifest"]["role_declarations"] = declarations;
10506        Frame::build(
10507            FrameType::Hello,
10508            control_flags(),
10509            0,
10510            0,
10511            301,
10512            serde_json::to_vec(&body).unwrap(),
10513        )
10514        .unwrap()
10515    }
10516
10517    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10518        let request = Frame::build(
10519            FrameType::Request,
10520            control_flags(),
10521            0,
10522            0,
10523            302,
10524            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10525        )
10526        .unwrap();
10527        let response = handler.handle_catalog_list(request, None).unwrap();
10528        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10529        let ClientControlResponse::CatalogList { mut modules, .. } =
10530            serde_json::from_slice(&response[0].body).unwrap()
10531        else {
10532            panic!("expected catalog.list");
10533        };
10534        (raw["modules"][0].clone(), modules.remove(0))
10535    }
10536
10537    #[test]
10538    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10539        let registry = Arc::new(Registry::default());
10540        let handler = ControlHandler::new(Arc::clone(&registry));
10541        let declarations = json!({
10542            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10543            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10544        });
10545        let connection = ConnectionId::new(301);
10546        let response = handler
10547            .handle_control(
10548                connection,
10549                hello_with_role_declarations(declarations.clone()),
10550            )
10551            .unwrap();
10552        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10553        let registered = registry
10554            .get_module_by_connection(connection)
10555            .unwrap()
10556            .unwrap()
10557            .manifest;
10558        // Compared as JSON values, not bytes: object key order is not part of
10559        // the relay's promise, and workspace builds unify serde_json's
10560        // preserve_order feature, which changes the order a literal keeps.
10561        assert_eq!(
10562            serde_json::to_value(&registered.role_declarations).unwrap(),
10563            declarations
10564        );
10565        let (raw, entry) = role_declarations_catalog(&handler);
10566        assert_eq!(raw["role_declarations"], declarations);
10567        assert_eq!(entry.role_declarations, registered.role_declarations);
10568
10569        let request =
10570            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10571        handler
10572            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10573            .unwrap();
10574        let (raw, entry) = role_declarations_catalog(&handler);
10575        assert_eq!(raw["role_declarations"], declarations);
10576        assert_eq!(entry.role_declarations, registered.role_declarations);
10577    }
10578
10579    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10580        let registry = Arc::new(Registry::default());
10581        let handler = ControlHandler::new(Arc::clone(&registry));
10582        let capture = EventCapture::default();
10583        let _subscriber =
10584            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10585        let response = handler
10586            .handle_control(
10587                ConnectionId::new(301),
10588                hello_with_role_declarations(declarations),
10589            )
10590            .unwrap();
10591        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10592        assert!(registry
10593            .get_module_by_connection(ConnectionId::new(301))
10594            .unwrap()
10595            .unwrap()
10596            .manifest
10597            .role_declarations
10598            .is_empty());
10599        let (raw, entry) = role_declarations_catalog(&handler);
10600        assert!(raw.get("role_declarations").is_none());
10601        assert!(entry.role_declarations.is_empty());
10602        let warnings: Vec<_> = capture
10603            .events()
10604            .into_iter()
10605            .filter(|event| {
10606                event.fields.get("message").map(String::as_str)
10607                    == Some("dropping whole role_declarations map from HELLO")
10608            })
10609            .collect();
10610        assert_eq!(warnings.len(), 1);
10611        assert_eq!(warnings[0].level, tracing::Level::WARN);
10612        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10613        assert_eq!(warnings[0].fields["reason"], reason);
10614    }
10615
10616    #[test]
10617    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10618        for value in [
10619            json!(null),
10620            json!(false),
10621            json!(0),
10622            json!("opaque"),
10623            json!([]),
10624        ] {
10625            assert_hello_drops_role_declarations_with_warning(
10626                json!({"routing/v1": {}, "broken/v1": value}),
10627                "role_declarations values must be objects",
10628            );
10629        }
10630        assert_hello_drops_role_declarations_with_warning(
10631            json!([]),
10632            "role_declarations must be an object",
10633        );
10634    }
10635
10636    #[test]
10637    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10638        assert_hello_drops_role_declarations_with_warning(
10639            json!({"routing/v1": {}, "bad key": {}}),
10640            "role_declarations key must use capability-name grammar",
10641        );
10642    }
10643
10644    #[test]
10645    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10646        assert_hello_drops_role_declarations_with_warning(
10647            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10648            "role_declarations exceeds 16384 compact JSON bytes",
10649        );
10650    }
10651
10652    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10653        let mut body = serde_json::to_value(ModuleHelloBody {
10654            manifest: manifest("flow-ops", PROTOCOL_VERSION),
10655            protocol_ver: PROTOCOL_VERSION,
10656            control_ops: None,
10657            launch_nonce: None,
10658        })
10659        .unwrap();
10660        body["manifest"]["provides"] = json!([{
10661            "role": "management_surface",
10662            "operations": [
10663                {"name": "diagnostics.findings", "kind": "query"},
10664                {"name": "reset", "kind": "mutate"}
10665            ],
10666            "config_schema": {}, "observability": [], "identity_scope": []
10667        }]);
10668        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10669        if let Some(flow_operations) = flow_operations {
10670            body["manifest"]["flow_operations"] = flow_operations;
10671        }
10672        Frame::build(
10673            FrameType::Hello,
10674            control_flags(),
10675            0,
10676            0,
10677            401,
10678            serde_json::to_vec(&body).unwrap(),
10679        )
10680        .unwrap()
10681    }
10682
10683    #[test]
10684    fn catalog_list_mirrors_flow_operations_unchanged() {
10685        // Order is kept as declared (not sorted) and an empty list stays an
10686        // affirmative "none", distinct from an absent declaration.
10687        for (capability, flow_operations) in [
10688            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10689            ("call-modes/v1", json!(["diagnostics.findings"])),
10690            ("flow-scopes/v1", json!([])),
10691        ] {
10692            let registry = Arc::new(Registry::default());
10693            let handler = ControlHandler::new(Arc::clone(&registry));
10694            let connection = ConnectionId::new(401);
10695            let response = handler
10696                .handle_control(
10697                    connection,
10698                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10699                )
10700                .unwrap();
10701            assert_eq!(response[0].header.ty, FrameType::HelloAck);
10702            let registered = registry
10703                .get_module_by_connection(connection)
10704                .unwrap()
10705                .unwrap()
10706                .manifest;
10707            assert_eq!(
10708                serde_json::to_value(&registered.flow_operations).unwrap(),
10709                flow_operations
10710            );
10711            let (raw, entry) = role_declarations_catalog(&handler);
10712            assert_eq!(raw["flow_operations"], flow_operations);
10713            assert_eq!(entry.flow_operations, registered.flow_operations);
10714        }
10715
10716        let registry = Arc::new(Registry::default());
10717        let handler = ControlHandler::new(Arc::clone(&registry));
10718        let response = handler
10719            .handle_control(
10720                ConnectionId::new(401),
10721                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10722            )
10723            .unwrap();
10724        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10725        let (raw, entry) = role_declarations_catalog(&handler);
10726        assert!(raw.get("flow_operations").is_none());
10727        assert_eq!(entry.flow_operations, None);
10728    }
10729
10730    #[test]
10731    fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
10732        let oversized: serde_json::Map<String, Value> = (0..257)
10733            .map(|n| (format!("tool:t{n}"), json!(true)))
10734            .collect();
10735        for (capabilities, map, valid) in [
10736            (
10737                json!(["scope-federated/v1"]),
10738                json!({"tool:read":true,"management:reset":true}),
10739                true,
10740            ),
10741            (json!(["scope-federated/v1"]), json!({}), true),
10742            (json!([]), json!({"tool:read":true}), false),
10743            (
10744                json!(["scope-federated/v1"]),
10745                json!({"tool:valid":true,"tool:bad":false}),
10746                false,
10747            ),
10748            (
10749                json!(["scope-federated/v1"]),
10750                json!({"tool:valid":true,"management:missing":true}),
10751                false,
10752            ),
10753            (
10754                json!(["scope-federated/v1"]),
10755                Value::Object(oversized),
10756                false,
10757            ),
10758        ] {
10759            let capture = EventCapture::default();
10760            let _subscriber = tracing::subscriber::set_default(
10761                tracing_subscriber::registry().with(capture.clone()),
10762            );
10763            let registry = Arc::new(Registry::default());
10764            let handler = ControlHandler::new(Arc::clone(&registry));
10765            let hello = hello_with_flow_operations(capabilities, None);
10766            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
10767            body["manifest"]["operation_deferral"] = map.clone();
10768            let hello = Frame::build(
10769                FrameType::Hello,
10770                control_flags(),
10771                0,
10772                0,
10773                401,
10774                serde_json::to_vec(&body).unwrap(),
10775            )
10776            .unwrap();
10777            let response = handler
10778                .handle_control(ConnectionId::new(401), hello)
10779                .unwrap();
10780            assert_eq!(response[0].header.ty, FrameType::HelloAck);
10781            let (raw, entry) = role_declarations_catalog(&handler);
10782            if valid {
10783                assert_eq!(raw["operation_deferral"], map);
10784                assert_eq!(
10785                    serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
10786                    map
10787                );
10788            } else {
10789                assert!(entry.operation_deferral.is_none());
10790                assert!(raw.get("operation_deferral").is_none());
10791                assert_eq!(
10792                    capture
10793                        .events()
10794                        .iter()
10795                        .filter(|event| event.fields.get("message").map(String::as_str)
10796                            == Some("dropping operation_deferral from HELLO"))
10797                        .count(),
10798                    1
10799                );
10800            }
10801        }
10802    }
10803
10804    fn assert_hello_drops_flow_operations_with_warning(
10805        capabilities: Value,
10806        flow_operations: Value,
10807        reason: &str,
10808    ) {
10809        let registry = Arc::new(Registry::default());
10810        let handler = ControlHandler::new(Arc::clone(&registry));
10811        let capture = EventCapture::default();
10812        let _subscriber =
10813            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10814        let response = handler
10815            .handle_control(
10816                ConnectionId::new(401),
10817                hello_with_flow_operations(capabilities, Some(flow_operations)),
10818            )
10819            .unwrap();
10820        assert_eq!(response[0].header.ty, FrameType::HelloAck);
10821        let registered = registry
10822            .get_module_by_connection(ConnectionId::new(401))
10823            .unwrap()
10824            .expect("the module still registers")
10825            .manifest;
10826        assert_eq!(registered.module_id, "flow-ops");
10827        assert_eq!(registered.flow_operations, None);
10828        let (raw, entry) = role_declarations_catalog(&handler);
10829        assert_eq!(raw["module_id"], "flow-ops");
10830        assert!(raw.get("flow_operations").is_none());
10831        assert_eq!(entry.flow_operations, None);
10832        let warnings: Vec<_> = capture
10833            .events()
10834            .into_iter()
10835            .filter(|event| {
10836                event.fields.get("message").map(String::as_str)
10837                    == Some("dropping flow_operations from HELLO")
10838            })
10839            .collect();
10840        assert_eq!(warnings.len(), 1);
10841        assert_eq!(warnings[0].level, tracing::Level::WARN);
10842        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10843        assert_eq!(warnings[0].fields["reason"], reason);
10844    }
10845
10846    #[test]
10847    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10848        assert_hello_drops_flow_operations_with_warning(
10849            json!(["flow-scopes/v1"]),
10850            json!(["diagnostics.findings", "missing"]),
10851            "flow_operations names 'missing', which is not a declared management operation",
10852        );
10853    }
10854
10855    #[test]
10856    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10857        assert_hello_drops_flow_operations_with_warning(
10858            json!([]),
10859            json!(["diagnostics.findings"]),
10860            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10861        );
10862        // A run-only capability does not admit flows either.
10863        assert_hello_drops_flow_operations_with_warning(
10864            json!(["agent-run-scopes/v1"]),
10865            json!([]),
10866            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10867        );
10868    }
10869
10870    #[test]
10871    fn hello_malformed_flow_operations_is_dropped_with_warning() {
10872        assert_hello_drops_flow_operations_with_warning(
10873            json!(["flow-scopes/v1"]),
10874            json!(["reset", "reset"]),
10875            "flow_operations names 'reset' more than once",
10876        );
10877        assert_hello_drops_flow_operations_with_warning(
10878            json!(["flow-scopes/v1"]),
10879            json!(vec!["reset"; 257]),
10880            "flow_operations lists 257 names, more than 256",
10881        );
10882        assert_hello_drops_flow_operations_with_warning(
10883            json!(["flow-scopes/v1"]),
10884            json!("reset"),
10885            "flow_operations must be an array of operation names",
10886        );
10887    }
10888
10889    #[test]
10890    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10891        let registry = Arc::new(Registry::default());
10892        let handler = ControlHandler::new(registry.clone());
10893        let descriptor = json!({
10894            "schema": "ckios.richui.manifest/1",
10895            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10896            "bytes": 128,
10897            "chunks": 2
10898        });
10899        let malformed = [
10900            ("schema", json!("")),
10901            ("schema", json!("a".repeat(65))),
10902            ("schema", json!("Bad schema")),
10903            (
10904                "manifest_digest",
10905                json!(format!("sha256:{}", "A".repeat(64))),
10906            ),
10907            ("manifest_digest", json!("sha256:abc")),
10908            ("bytes", json!(0)),
10909            ("bytes", json!(16_777_217)),
10910            ("bytes", json!("128")),
10911            ("chunks", json!(0)),
10912            ("chunks", json!(1025)),
10913            ("chunks", json!("2")),
10914        ];
10915        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10916            let mut invalid = descriptor.clone();
10917            invalid[field] = invalid_value;
10918            let response = handler
10919                .handle_control(
10920                    ConnectionId::new(210 + index as u64),
10921                    hello_frame_with_rich_ui(
10922                        &format!("bad-rich-ui-{index}"),
10923                        PROTOCOL_VERSION,
10924                        210 + index as u64,
10925                        None,
10926                        invalid,
10927                    ),
10928                )
10929                .unwrap();
10930            assert_eq!(response[0].header.ty, FrameType::Error);
10931            let error = parse_error(&response[0]);
10932            assert_eq!(error["code"], "invalid_manifest");
10933            assert!(error["message"]
10934                .as_str()
10935                .unwrap()
10936                .contains(&format!("rich_ui.{field}")));
10937            assert!(registry
10938                .get_module(&format!("bad-rich-ui-{index}"))
10939                .unwrap()
10940                .is_none());
10941        }
10942    }
10943
10944    #[test]
10945    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10946        let registry = Arc::new(Registry::default());
10947        let handler = ControlHandler::new(Arc::clone(&registry));
10948        let connection = ConnectionId::new(219);
10949        registry
10950            .register_with_control_ops(
10951                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10952                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10953                        "bad schema",
10954                        format!("sha256:{}", "a".repeat(64)),
10955                        128,
10956                        2,
10957                    )))
10958                    .build(),
10959                PROTOCOL_VERSION,
10960                connection,
10961                Vec::new(),
10962            )
10963            .unwrap();
10964        let request =
10965            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10966        let response = handler
10967            .handle_catalog_update(connection, request, Vec::new(), None, None)
10968            .unwrap();
10969        assert_eq!(response[0].header.ty, FrameType::Error);
10970        let error = parse_error(&response[0]);
10971        assert_eq!(error["code"], "invalid_manifest");
10972        assert!(error["message"]
10973            .as_str()
10974            .unwrap()
10975            .contains("rich_ui.schema"));
10976    }
10977
10978    #[test]
10979    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10980        let registry = Arc::new(Registry::default());
10981        let supervisor = SupervisorHandle::new();
10982        supervisor.open_swap("provider", "candidate-nonce".to_string());
10983        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10984        let connection = ConnectionId::new(218);
10985        let hello = handler
10986            .handle_control(
10987                connection,
10988                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10989            )
10990            .unwrap();
10991        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10992        assert!(registry.list_modules().unwrap().1.is_empty());
10993
10994        registry.promote_candidate("provider").unwrap().unwrap();
10995        let request = Frame::build(
10996            FrameType::Request,
10997            control_flags(),
10998            0,
10999            0,
11000            219,
11001            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11002        )
11003        .unwrap();
11004        let response = handler.handle_catalog_list(request, None).unwrap();
11005        let ClientControlResponse::CatalogList { modules, .. } =
11006            serde_json::from_slice(&response[0].body).unwrap()
11007        else {
11008            panic!("expected catalog.list");
11009        };
11010        assert_eq!(modules.len(), 1);
11011        assert!(modules[0].nonce_verified);
11012    }
11013
11014    #[test]
11015    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
11016        for provenance_json in [
11017            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
11018            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
11019        ] {
11020            let registry = Arc::new(Registry::default());
11021            let handler = ControlHandler::new(registry);
11022            let hello = Frame::build(
11023                FrameType::Hello,
11024                control_flags(),
11025                0,
11026                0,
11027                220,
11028                serde_json::to_vec(&ModuleHelloBody {
11029                    manifest: ModuleManifest::builder("built", "0.1.0")
11030                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
11031                        .build(),
11032                    protocol_ver: PROTOCOL_VERSION,
11033                    control_ops: None,
11034                    launch_nonce: None,
11035                })
11036                .unwrap(),
11037            )
11038            .unwrap();
11039            assert_eq!(
11040                handler
11041                    .handle_control(ConnectionId::new(220), hello)
11042                    .unwrap()[0]
11043                    .header
11044                    .ty,
11045                FrameType::HelloAck
11046            );
11047            let request = Frame::build(
11048                FrameType::Request,
11049                control_flags(),
11050                0,
11051                0,
11052                221,
11053                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11054            )
11055            .unwrap();
11056            let response = handler.handle_catalog_list(request, None).unwrap();
11057            let ClientControlResponse::CatalogList { modules, .. } =
11058                serde_json::from_slice(&response[0].body).unwrap()
11059            else {
11060                panic!("expected catalog.list");
11061            };
11062            assert_eq!(modules.len(), 1);
11063            assert_eq!(
11064                serde_json::to_vec(&modules[0]).unwrap(),
11065                format!(
11066                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
11067                )
11068                .as_bytes()
11069            );
11070        }
11071    }
11072
11073    #[test]
11074    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
11075        let registry = Arc::new(Registry::default());
11076        let handler = ControlHandler::new(Arc::clone(&registry));
11077        let hello = Frame::build(
11078            FrameType::Hello,
11079            control_flags(),
11080            0,
11081            0,
11082            220,
11083            serde_json::to_vec(&ModuleHelloBody {
11084                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
11085                protocol_ver: PROTOCOL_VERSION,
11086                control_ops: None,
11087                launch_nonce: Some("unverified".to_string()),
11088            })
11089            .unwrap(),
11090        )
11091        .unwrap();
11092        assert_eq!(
11093            handler
11094                .handle_control(ConnectionId::new(220), hello)
11095                .unwrap()[0]
11096                .header
11097                .ty,
11098            FrameType::HelloAck
11099        );
11100        let request = Frame::build(
11101            FrameType::Request,
11102            control_flags(),
11103            0,
11104            0,
11105            221,
11106            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11107        )
11108        .unwrap();
11109        let response = handler.handle_catalog_list(request, None).unwrap();
11110        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
11111        assert!(wire["modules"][0].get("provenance").is_none());
11112        let ClientControlResponse::CatalogList { modules, .. } =
11113            serde_json::from_slice(&response[0].body).unwrap()
11114        else {
11115            panic!("expected catalog.list");
11116        };
11117        assert_eq!(
11118            serde_json::to_vec(&modules[0]).unwrap(),
11119            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
11120        );
11121        assert!(!modules[0].nonce_verified);
11122        assert!(modules[0].provenance.is_none());
11123    }
11124
11125    #[test]
11126    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
11127        let registry = Arc::new(Registry::default());
11128        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
11129            [("vault".to_string(), true), ("squatter".to_string(), true)],
11130            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11131        );
11132        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
11133        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11134            provides: vec!["credentials-provider/v1".to_string()],
11135            requires: Vec::new(),
11136            must_never_reach: Vec::new(),
11137        });
11138        let frame = Frame::build(
11139            FrameType::Hello,
11140            control_flags(),
11141            0,
11142            0,
11143            77,
11144            serde_json::to_vec(&ModuleHelloBody {
11145                manifest: squatter,
11146                protocol_ver: PROTOCOL_VERSION,
11147                control_ops: None,
11148                launch_nonce: None,
11149            })
11150            .expect("HELLO serializes"),
11151        )
11152        .expect("HELLO frame builds");
11153        let response = handler
11154            .handle_control(ConnectionId::new(77), frame)
11155            .expect("reserved claim receives a typed refusal");
11156        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11157        assert_eq!(
11158            registry
11159                .active_registration_count()
11160                .expect("registry reads"),
11161            0,
11162            "a reserved capability refusal must not leave a catalog entry"
11163        );
11164    }
11165
11166    #[test]
11167    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
11168        for settlement in ["timeout", "inconclusive", "drop"] {
11169            let breakers = RouteBindBreakers::default();
11170            let RouteBindAdmission::Admitted {
11171                guard: mut old,
11172                probe: false,
11173            } = breakers.admit("prov")
11174            else {
11175                panic!("ordinary relay admitted")
11176            };
11177            let RouteBindAdmission::Admitted {
11178                guard: mut opener, ..
11179            } = breakers.admit("prov")
11180            else {
11181                panic!("second relay admitted")
11182            };
11183            assert!(
11184                !opener
11185                    .record_timeout(1, Duration::ZERO)
11186                    .unwrap()
11187                    .reopened_after_probe
11188            );
11189            let RouteBindAdmission::Admitted {
11190                guard: mut probe,
11191                probe: true,
11192            } = breakers.admit("prov")
11193            else {
11194                panic!("one cooldown probe admitted")
11195            };
11196            match settlement {
11197                "timeout" => assert!(
11198                    !old.record_timeout(1, Duration::ZERO)
11199                        .unwrap()
11200                        .reopened_after_probe
11201                ),
11202                "inconclusive" => old.record_inconclusive(),
11203                "drop" => drop(old),
11204                _ => unreachable!(),
11205            }
11206            assert!(
11207                matches!(
11208                    breakers.admit("prov"),
11209                    RouteBindAdmission::Refused {
11210                        probe_in_flight: true,
11211                        ..
11212                    }
11213                ),
11214                "{settlement} of a pre-open relay cannot release the real probe"
11215            );
11216            assert!(
11217                probe
11218                    .record_timeout(1, Duration::ZERO)
11219                    .unwrap()
11220                    .reopened_after_probe
11221            );
11222            assert!(matches!(
11223                breakers.admit("prov"),
11224                RouteBindAdmission::Admitted { probe: true, .. }
11225            ));
11226        }
11227        let breakers = RouteBindBreakers::default();
11228        let admit = || match breakers.admit("prov") {
11229            RouteBindAdmission::Admitted { guard, .. } => guard,
11230            _ => panic!("relay admitted"),
11231        };
11232        admit().record_timeout(1, Duration::ZERO);
11233        let mut old_probe = admit();
11234        breakers.reset_for_new_module_connection("prov");
11235        admit().record_timeout(1, Duration::ZERO);
11236        let _new_probe = admit();
11237        old_probe.record_inconclusive();
11238        assert!(matches!(
11239            breakers.admit("prov"),
11240            RouteBindAdmission::Refused {
11241                probe_in_flight: true,
11242                ..
11243            }
11244        ));
11245    }
11246
11247    #[tokio::test]
11248    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11249        for candidate in [false, true] {
11250            let registry = Arc::new(Registry::default());
11251            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
11252                [("vault".to_string(), true), ("squatter".to_string(), true)],
11253                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11254            );
11255            let conn = ConnectionId::new(77);
11256            let (ctx, mut rx) = route_ctx(conn);
11257            let initial = capability_manifest("squatter", &[], &[]);
11258            if candidate {
11259                registry
11260                    .register_candidate_with_control_ops(
11261                        initial.clone(),
11262                        PROTOCOL_VERSION,
11263                        conn,
11264                        module_baseline_control_ops(),
11265                    )
11266                    .unwrap();
11267                handler
11268                    .forwarding
11269                    .register_candidate_module_connection(
11270                        conn,
11271                        "squatter".to_string(),
11272                        PROTOCOL_VERSION,
11273                        manifest_concurrency(&initial),
11274                        ctx.egress.clone(),
11275                    )
11276                    .unwrap();
11277            } else {
11278                hello_via_sink(
11279                    &handler,
11280                    &ctx,
11281                    &mut rx,
11282                    hello_frame_with_manifest(initial.clone(), 1),
11283                )
11284                .await;
11285            }
11286            let response = handler
11287                .handle_control_frame(
11288                    &ctx,
11289                    catalog_update_with_capabilities_frame(
11290                        2,
11291                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
11292                            .capabilities
11293                            .unwrap(),
11294                    ),
11295                )
11296                .await
11297                .unwrap();
11298            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11299            assert_eq!(
11300                registry
11301                    .get_module_by_connection(conn)
11302                    .unwrap()
11303                    .unwrap()
11304                    .manifest,
11305                initial
11306            );
11307        }
11308    }
11309
11310    #[test]
11311    fn server_describe_surfaces_required_capability_verdict_fields() {
11312        let registry = Arc::new(Registry::default());
11313        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
11314            [
11315                ("consumer".to_string(), true),
11316                ("provider".to_string(), false),
11317            ],
11318            BTreeMap::new(),
11319        );
11320        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11321        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11322            provides: Vec::new(),
11323            requires: vec![subc_protocol::manifest::CapabilityRequirement {
11324                capability: "credentials-provider/v1".to_string(),
11325                need: subc_protocol::manifest::CapabilityNeed::Required,
11326            }],
11327            must_never_reach: Vec::new(),
11328        });
11329        let hello = Frame::build(
11330            FrameType::Hello,
11331            control_flags(),
11332            0,
11333            0,
11334            78,
11335            serde_json::to_vec(&ModuleHelloBody {
11336                manifest: consumer,
11337                protocol_ver: PROTOCOL_VERSION,
11338                control_ops: None,
11339                launch_nonce: None,
11340            })
11341            .expect("HELLO serializes"),
11342        )
11343        .expect("HELLO frame builds");
11344        handler
11345            .handle_control(ConnectionId::new(78), hello)
11346            .expect("consumer registers");
11347        let describe = Frame::build(
11348            FrameType::Request,
11349            control_flags(),
11350            0,
11351            0,
11352            79,
11353            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11354                .expect("request serializes"),
11355        )
11356        .expect("describe frame builds");
11357        let response = handler
11358            .handle_server_describe(describe)
11359            .expect("server.describe succeeds");
11360        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11361        let requirement = &rendered["capability_requirements"][0];
11362        assert_eq!(requirement["consumer"], "consumer");
11363        assert_eq!(requirement["verdict"], "never_provided");
11364        assert_eq!(requirement["episode_seq"], 1);
11365        assert_eq!(requirement["config_satisfiable"], false);
11366        assert_eq!(requirement["runtime_available"], false);
11367        assert!(requirement["detail"]
11368            .as_str()
11369            .expect("detail string")
11370            .contains("credentials-provider/v1"));
11371    }
11372
11373    /// Register a consumer that requires `project-identity/v1` and the provider
11374    /// of it, optionally mark daemon shutdown as begun, then deregister the
11375    /// provider as its connection closes. Returns the `capability.requirement`
11376    /// events that deregistration logged and the verdict the evaluator then
11377    /// holds for the requirement (the one `route.open` refusals read).
11378    fn deregister_required_provider(
11379        daemon_shutdown_begun: bool,
11380    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11381        const CAPABILITY: &str = "project-identity/v1";
11382        let registry = Arc::new(Registry::default());
11383        let daemon_shutdown = DaemonShutdownFlag::default();
11384        let handler = ControlHandler::new(Arc::clone(&registry))
11385            .with_capability_config(
11386                [
11387                    ("consumer".to_string(), true),
11388                    ("provider".to_string(), true),
11389                ],
11390                BTreeMap::new(),
11391            )
11392            .with_daemon_shutdown_flag(daemon_shutdown.clone());
11393        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11394        consumer.capabilities = Some(CapabilityDeclarations {
11395            provides: Vec::new(),
11396            requires: vec![subc_protocol::manifest::CapabilityRequirement {
11397                capability: CAPABILITY.to_string(),
11398                need: subc_protocol::manifest::CapabilityNeed::Required,
11399            }],
11400            must_never_reach: Vec::new(),
11401        });
11402        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11403        let provider_conn = ConnectionId::new(182);
11404        for (conn, manifest, corr) in [
11405            (ConnectionId::new(181), consumer, 181),
11406            (provider_conn, provider, 182),
11407        ] {
11408            let hello = handler
11409                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11410                .expect("module registers");
11411            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11412        }
11413        assert_eq!(
11414            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11415            Some(CapabilityVerdict::Provided),
11416            "the registered provider must satisfy the requirement first"
11417        );
11418        if daemon_shutdown_begun {
11419            daemon_shutdown.set_for_test();
11420        }
11421
11422        let capture = EventCapture::default();
11423        let _subscriber =
11424            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11425        handler
11426            .cleanup_connection(provider_conn)
11427            .expect("provider deregisters");
11428        let events = capture
11429            .events()
11430            .into_iter()
11431            .filter(|event| {
11432                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11433            })
11434            .collect();
11435        (
11436            events,
11437            handler.capability_evaluator.verdict("consumer", CAPABILITY),
11438        )
11439    }
11440
11441    #[test]
11442    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11443        let (events, verdict) = deregister_required_provider(true);
11444        assert_eq!(
11445            verdict,
11446            Some(CapabilityVerdict::NeverProvided),
11447            "shutdown must not change the verdict route.open refusals read"
11448        );
11449        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11450        let event = &events[0];
11451        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11452        assert_eq!(
11453            event.fields.get("cause").map(String::as_str),
11454            Some("\"daemon_shutdown\""),
11455            "{event:?}"
11456        );
11457        assert_eq!(
11458            event.fields.get("verdict").map(String::as_str),
11459            Some("\"never_provided\""),
11460            "{event:?}"
11461        );
11462    }
11463
11464    #[test]
11465    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11466        let (events, verdict) = deregister_required_provider(false);
11467        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11468        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11469        let event = &events[0];
11470        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11471        assert_eq!(event.fields.get("cause"), None, "{event:?}");
11472        assert_eq!(
11473            event.fields.get("verdict").map(String::as_str),
11474            Some("\"never_provided\""),
11475            "{event:?}"
11476        );
11477    }
11478
11479    #[test]
11480    fn catalog_list_omits_capabilities_for_legacy_manifest() {
11481        let registry = Arc::new(Registry::default());
11482        let handler = ControlHandler::new(Arc::clone(&registry));
11483        let hello = handler
11484            .handle_control(
11485                ConnectionId::new(101),
11486                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11487            )
11488            .expect("legacy HELLO registers");
11489        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11490
11491        let request = Frame::build(
11492            FrameType::Request,
11493            control_flags(),
11494            0,
11495            0,
11496            102,
11497            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11498                .expect("catalog request serializes"),
11499        )
11500        .expect("catalog request frame builds");
11501        let response = handler
11502            .handle_catalog_list(request, None)
11503            .expect("catalog list succeeds");
11504        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11505        assert!(
11506            body["modules"][0].get("capabilities").is_none(),
11507            "legacy manifest must retain an absent capabilities field on catalog.list"
11508        );
11509    }
11510
11511    #[test]
11512    fn hello_ack_omits_storage_when_no_storage_config() {
11513        let registry = Arc::new(Registry::default());
11514        let handler = ControlHandler::new(Arc::clone(&registry));
11515        let responses = handler
11516            .handle_control(
11517                ConnectionId::new(1),
11518                hello_frame("aft", PROTOCOL_VERSION, 7),
11519            )
11520            .unwrap();
11521        let ack = parse_ack(&responses[0]);
11522        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11523        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11524    }
11525
11526    #[tokio::test]
11527    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11528        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11529        let registry = Arc::new(Registry::default());
11530        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
11531        let responses = handler
11532            .handle_control(
11533                ConnectionId::new(1),
11534                hello_frame("aft", PROTOCOL_VERSION, 7),
11535            )
11536            .unwrap();
11537        let ack = parse_ack(&responses[0]);
11538        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11539
11540        let described = handler
11541            .handle_control_frame(
11542                &route_ctx(ConnectionId::new(2)).0,
11543                Frame::build(
11544                    FrameType::Request,
11545                    control_flags(),
11546                    0,
11547                    0,
11548                    9,
11549                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11550                )
11551                .unwrap(),
11552            )
11553            .await
11554            .unwrap();
11555        let ClientControlResponse::ServerDescribe { machine_id, .. } =
11556            serde_json::from_slice(&described[0].body).unwrap()
11557        else {
11558            panic!("server.describe answered with another shape");
11559        };
11560        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11561    }
11562
11563    #[test]
11564    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11565        // With a central sqlite storage policy, each registering module gets its
11566        // own resolved descriptor in HELLO_ACK, keyed by its module id.
11567        let registry = Arc::new(Registry::default());
11568        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
11569            crate::daemon_config::StorageConfig::Sqlite {
11570                data_home: std::path::PathBuf::from("/data"),
11571            },
11572        ));
11573
11574        let responses = handler
11575            .handle_control(
11576                ConnectionId::new(1),
11577                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11578            )
11579            .unwrap();
11580        let ack = parse_ack(&responses[0]);
11581        assert_eq!(
11582            ack.storage,
11583            Some(serde_json::json!({
11584                "module_id": "alfonso-routing",
11585                "storage_namespace": "default",
11586                "isolation": { "kind": "module" },
11587                "backend": {
11588                    "backend": "sqlite",
11589                    "path": "/data/cortexkit/alfonso-routing/store.db"
11590                }
11591            })),
11592            "the delivered descriptor is the module's own sqlite store path"
11593        );
11594    }
11595
11596    #[test]
11597    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11598        let registry = Arc::new(Registry::default());
11599        let handler = ControlHandler::new(Arc::clone(&registry));
11600        let conn = ConnectionId::new(1);
11601        let responses = handler
11602            .handle_control(
11603                conn,
11604                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11605            )
11606            .unwrap();
11607        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11608        let registration = registry.get_module("aft").unwrap().unwrap();
11609        assert_eq!(registration.control_ops, module_baseline_control_ops());
11610
11611        let frame =
11612            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11613        assert!(handler
11614            .guard_module_control_op(&frame, "aft", "route.bind")
11615            .unwrap()
11616            .is_none());
11617        let error = handler
11618            .guard_module_control_op(&frame, "aft", "test.synthetic")
11619            .unwrap()
11620            .expect("synthetic ungranted op should be rejected");
11621        assert_eq!(error.header.ty, FrameType::Error);
11622        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11623    }
11624
11625    #[test]
11626    fn hello_control_ops_some_adds_optional_grants() {
11627        let registry = Arc::new(Registry::default());
11628        let handler = ControlHandler::new(Arc::clone(&registry));
11629        handler
11630            .handle_control(
11631                ConnectionId::new(1),
11632                hello_frame_with_control_ops(
11633                    "aft",
11634                    PROTOCOL_VERSION,
11635                    7,
11636                    Some(vec![
11637                        "future.synthetic".to_string(),
11638                        "route.bind".to_string(),
11639                    ]),
11640                ),
11641            )
11642            .unwrap();
11643        let registration = registry.get_module("aft").unwrap().unwrap();
11644        assert_eq!(
11645            registration.control_ops,
11646            vec![
11647                "route.bind".to_string(),
11648                "route.status".to_string(),
11649                "future.synthetic".to_string(),
11650            ]
11651        );
11652        let frame =
11653            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11654        assert!(handler
11655            .guard_module_control_op(&frame, "aft", "future.synthetic")
11656            .unwrap()
11657            .is_none());
11658    }
11659
11660    #[tokio::test]
11661    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11662        let registry = Arc::new(Registry::default());
11663        let forwarding = Arc::new(ForwardingTable::default());
11664        let handler =
11665            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11666        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11667        hello_via_sink(
11668            &handler,
11669            &module_ctx,
11670            &mut module_rx,
11671            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11672        )
11673        .await;
11674
11675        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11676        let responses = handler
11677            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11678            .await
11679            .unwrap();
11680        assert_eq!(responses.len(), 1);
11681        assert_eq!(responses[0].header.ty, FrameType::Error);
11682        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11683        assert!(module_rx.try_recv().is_err());
11684    }
11685
11686    #[tokio::test]
11687    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11688        let registry = Arc::new(Registry::default());
11689        let forwarding = Arc::new(ForwardingTable::default());
11690        let handler =
11691            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11692        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11693        hello_via_sink(
11694            &handler,
11695            &module_ctx,
11696            &mut module_rx,
11697            hello_frame_with_control_ops(
11698                "aft",
11699                PROTOCOL_VERSION,
11700                7,
11701                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11702            ),
11703        )
11704        .await;
11705
11706        let project_root = unique_project_root("demux");
11707        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11708        let route_handler = handler.clone();
11709        let route_task = tokio::spawn(async move {
11710            route_handler
11711                .handle_control_frame(
11712                    &route_client_ctx,
11713                    route_open_frame(100, "aft", project_root),
11714                )
11715                .await
11716                .unwrap()
11717        });
11718        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11719            .await
11720            .unwrap()
11721            .unwrap();
11722        assert!(matches!(
11723            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11724            ModuleControlRequest::RouteBind { .. }
11725        ));
11726
11727        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11728        let health_handler = handler.clone();
11729        let health_task = tokio::spawn(async move {
11730            health_handler
11731                .handle_control_frame(
11732                    &health_client_ctx,
11733                    supervisor_health_probe_frame(101, "aft"),
11734                )
11735                .await
11736                .unwrap()
11737        });
11738        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11739            .await
11740            .unwrap()
11741            .unwrap();
11742        assert_eq!(
11743            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11744            ModuleControlRequest::HealthCheck {}
11745        );
11746
11747        handler
11748            .handle_control_frame(
11749                &module_ctx,
11750                health_response(health_frame.header.corr, HealthStatus::Degraded),
11751            )
11752            .await
11753            .unwrap();
11754        let health_response = health_task.await.unwrap();
11755        assert_eq!(health_response.len(), 1);
11756        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11757            ClientControlResponse::SupervisorHealthProbe {
11758                module_id,
11759                status,
11760                detail,
11761                metrics,
11762            } => {
11763                assert_eq!(module_id, "aft");
11764                assert_eq!(status, HealthStatus::Degraded);
11765                assert_eq!(detail.as_deref(), Some("warming"));
11766                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11767            }
11768            other => panic!("unexpected health response: {other:?}"),
11769        }
11770
11771        handler
11772            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11773            .await
11774            .unwrap();
11775        let route_response = route_task.await.unwrap();
11776        assert!(route_response.is_empty());
11777        let published = route_client_rx.recv().await.unwrap();
11778        assert!(matches!(
11779            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11780            ClientControlResponse::RouteOpen { .. }
11781        ));
11782    }
11783
11784    /// Start one `route.open` on `client_connection` and return its still-running
11785    /// handler task together with the `route.bind` the module received for it.
11786    /// The handler blocks until the module answers, so it has to run as a task
11787    /// while the test drives the module side.
11788    async fn relay_route_open(
11789        handler: &ControlHandler,
11790        client_connection: ConnectionId,
11791        client_egress: &FrameSink,
11792        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11793        corr: u64,
11794        module_id: &str,
11795        project_root_label: &str,
11796    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11797        let ctx = RouteCtx {
11798            connection_id: client_connection,
11799            egress: client_egress.clone(),
11800        };
11801        let handler = handler.clone();
11802        let project_root = unique_project_root(project_root_label);
11803        let module_id = module_id.to_string();
11804        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11805        let task = tokio::spawn(async move {
11806            let _guard = tracing::dispatcher::set_default(&dispatch);
11807            handler
11808                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11809                .await
11810                .unwrap()
11811        });
11812        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11813            .await
11814            .expect("module receives the relayed route.bind")
11815            .expect("module egress is open");
11816        (task, bind.frame)
11817    }
11818
11819    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11820        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11821            ModuleControlRequest::RouteBind {
11822                route_channel,
11823                epoch,
11824                ..
11825            } => (route_channel, epoch),
11826            other => panic!("expected a route.bind request, got {other:?}"),
11827        }
11828    }
11829
11830    fn published_route(frame: &Frame) -> (u16, u32) {
11831        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11832            ClientControlResponse::RouteOpen {
11833                route_channel,
11834                route_epoch,
11835                ..
11836            } => (route_channel, route_epoch),
11837            other => panic!("expected a route.open response, got {other:?}"),
11838        }
11839    }
11840
11841    /// Reproduction of a production outage. A client had `route.open`s in
11842    /// flight to a module and was already marked closing -- its egress had refused a
11843    /// module frame, so the daemon asked its connection to end -- while its sink
11844    /// was still open. When the module acked those binds, the daemon refused to
11845    /// commit a route for a closing client, and that refusal was returned from
11846    /// the MODULE connection's frame handler, where a router error that has no
11847    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
11848    /// the supervisor correctly did not respawn a clean exit, and every seat lost
11849    /// its tools for hours -- one client's teardown took down a connection
11850    /// carrying ~170 other routes.
11851    ///
11852    /// The window is opened here by calling the production path that opens it
11853    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
11854    /// state that matters is "in `closing_connections`, sink still open, relay
11855    /// still pending", and it lasts only from the close request until the
11856    /// connection loop reacts to it; a socket-level test can flood a client into
11857    /// that escalation but cannot pin the module's ack inside the window. Closing
11858    /// the socket instead takes the other path entirely -- connection teardown
11859    /// removes the pending relay under the same lock, so the ack finds nothing.
11860    #[tokio::test]
11861    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11862        let registry = Arc::new(Registry::default());
11863        let forwarding = Arc::new(ForwardingTable::default());
11864        let handler =
11865            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11866
11867        let module_connection = ConnectionId::new(30);
11868        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11869        hello_via_sink(
11870            &handler,
11871            &module_ctx,
11872            &mut module_rx,
11873            hello_frame("aft", PROTOCOL_VERSION, 7),
11874        )
11875        .await;
11876
11877        let dying_client = ConnectionId::new(31);
11878        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11879
11880        // A published route on the dying client. The escalation below only marks
11881        // a connection closing for a route it has already published.
11882        let (first_task, first_bind) = relay_route_open(
11883            &handler,
11884            dying_client,
11885            &dying_ctx.egress,
11886            &mut module_rx,
11887            100,
11888            "aft",
11889            "closing-first",
11890        )
11891        .await;
11892        handler
11893            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11894            .await
11895            .unwrap();
11896        assert!(first_task.await.unwrap().is_empty());
11897        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11898
11899        // A second route.open from the same client, relayed and awaiting its ack.
11900        let (second_task, second_bind) = relay_route_open(
11901            &handler,
11902            dying_client,
11903            &dying_ctx.egress,
11904            &mut module_rx,
11905            101,
11906            "aft",
11907            "closing-second",
11908        )
11909        .await;
11910        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11911
11912        // The window: the client is closing, its sink is still open, and its
11913        // second bind is still pending.
11914        assert!(forwarding
11915            .escalate_client_delivery_failure(
11916                dying_client,
11917                first_channel,
11918                first_epoch,
11919                CloseReason::new(
11920                    "module_to_client_delivery_failed",
11921                    "client egress refused a module frame",
11922                ),
11923                crate::forwarding::UndeliveredFrame {
11924                    module_id: None,
11925                    sink: &dying_ctx.egress,
11926                },
11927            )
11928            .unwrap());
11929        assert!(!dying_ctx.egress.is_closed());
11930
11931        // The frame that used to end the module connection.
11932        let ack = handler
11933            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11934            .await;
11935        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11936        if module_loop_error.is_some() {
11937            // What the server's connection loop does with a router error that has
11938            // no ERROR-frame translation: end the connection, which releases the
11939            // module's registration and every route on it.
11940            handler.cleanup_connection(module_connection).unwrap();
11941        }
11942        // Read the module's next frame before opening the co-tenant's route, so
11943        // the GOODBYE assertion below is about THIS ack and not about later
11944        // traffic. `None` means the module was told nothing.
11945        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11946            .await
11947            .ok()
11948            .flatten();
11949
11950        // 1. The module connection is still registered.
11951        assert!(
11952            registry
11953                .get_module_by_connection(module_connection)
11954                .unwrap()
11955                .is_some(),
11956            "one client's closing connection ended the shared module connection: \
11957             {module_loop_error:?}"
11958        );
11959        // ...and still serving: another client can open and use a route on it.
11960        let cotenant = ConnectionId::new(32);
11961        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11962        let (cotenant_task, cotenant_bind) = relay_route_open(
11963            &handler,
11964            cotenant,
11965            &cotenant_ctx.egress,
11966            &mut module_rx,
11967            102,
11968            "aft",
11969            "closing-cotenant",
11970        )
11971        .await;
11972        handler
11973            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11974            .await
11975            .unwrap();
11976        assert!(cotenant_task.await.unwrap().is_empty());
11977        let (cotenant_channel, cotenant_epoch) =
11978            published_route(&cotenant_rx.recv().await.unwrap());
11979        assert!(matches!(
11980            forwarding
11981                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11982                .unwrap(),
11983            DataRoute::Client(DataRouteState::Bound(_))
11984        ));
11985
11986        // 2. The module was told to drop the binding it created for the route
11987        //    that will never be published.
11988        let goodbye = post_ack_module_frame
11989            .expect("module receives a GOODBYE for the abandoned route channel");
11990        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11991        assert_eq!(goodbye.header.channel, abandoned_channel);
11992        assert_eq!(goodbye.header.epoch, abandoned_epoch);
11993
11994        // 3. The dying client received nothing: no route was ever published to
11995        //    it. Its route.open is answered as unavailable, which the connection
11996        //    loop would write to a socket that is already going away.
11997        assert!(dying_rx.try_recv().is_err());
11998        let second_response = second_task.await.unwrap();
11999        assert_eq!(second_response.len(), 1);
12000        assert_eq!(
12001            parse_error(&second_response[0])["code"],
12002            "target_unavailable"
12003        );
12004    }
12005
12006    /// One `catalog.list` as a client sees it: the generation and the listed
12007    /// module ids with their readiness.
12008    async fn catalog_snapshot(
12009        handler: &ControlHandler,
12010        client_ctx: &RouteCtx,
12011        corr: u64,
12012    ) -> (u64, Vec<(String, bool)>) {
12013        let body =
12014            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
12015        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
12016        let reply = handler
12017            .handle_control_frame(client_ctx, request)
12018            .await
12019            .unwrap()
12020            .pop()
12021            .unwrap();
12022        let ClientControlResponse::CatalogList {
12023            generation,
12024            modules,
12025            ..
12026        } = serde_json::from_slice(&reply.body).unwrap()
12027        else {
12028            panic!("not a catalog.list reply");
12029        };
12030        (
12031            generation,
12032            modules
12033                .into_iter()
12034                .map(|entry| (entry.module_id, entry.ready))
12035                .collect(),
12036        )
12037    }
12038
12039    /// A handler whose HELLO for module "aft" on connection 40 is parked
12040    /// between reserving its registry entry and installing its forwarding
12041    /// endpoint. Returns the handler, the two-party barrier (wait once to know
12042    /// the HELLO is parked, once more to release it), the HELLO's thread, and
12043    /// the module's context and receiver.
12044    fn hello_held_before_forwarding() -> (
12045        ControlHandler,
12046        Arc<std::sync::Barrier>,
12047        std::thread::JoinHandle<Vec<Frame>>,
12048        RouteCtx,
12049        mpsc::Receiver<crate::router::OutboundFrame>,
12050    ) {
12051        let hold = Arc::new(std::sync::Barrier::new(2));
12052        let handler = ControlHandler::with_forwarding(
12053            Arc::new(Registry::default()),
12054            Arc::new(ForwardingTable::default()),
12055        )
12056        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
12057        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12058        let hello = {
12059            let handler = handler.clone();
12060            let module_ctx = module_ctx.clone();
12061            std::thread::spawn(move || {
12062                handler
12063                    .handle_hello(
12064                        module_ctx.connection_id,
12065                        Some(module_ctx.egress.clone()),
12066                        hello_frame("aft", PROTOCOL_VERSION, 7),
12067                    )
12068                    .unwrap()
12069            })
12070        };
12071        (handler, hold, hello, module_ctx, module_rx)
12072    }
12073
12074    /// While a HELLO has reserved its registry entry but its forwarding
12075    /// endpoint is not installed yet, `route.open` to the module can only be
12076    /// refused. `catalog.list` must not list the module in that window: a
12077    /// client that opens a route as soon as it sees a module listed and ready
12078    /// would be refused for a module the daemon had just advertised. The
12079    /// refusal in the window stays the retryable `target_unavailable`, never
12080    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
12081    /// Once the HELLO completes, the module is listed and a route to it opens.
12082    #[tokio::test]
12083    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
12084        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
12085        let client = ConnectionId::new(41);
12086        let (client_ctx, mut client_rx) = route_ctx(client);
12087
12088        hold.wait();
12089        // Parked: the registry holds "aft", forwarding has no endpoint for it.
12090        assert!(handler.registry.get_module("aft").unwrap().is_some());
12091        assert!(!handler
12092            .forwarding
12093            .has_live_module_connection("aft")
12094            .unwrap());
12095        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
12096        assert!(
12097            listed.is_empty(),
12098            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
12099        );
12100        let refused = handler
12101            .handle_control_frame(
12102                &client_ctx,
12103                route_open_frame(301, "aft", unique_project_root("listed-window")),
12104            )
12105            .await
12106            .unwrap();
12107        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
12108        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
12109        let duplicate = handler
12110            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
12111            .await
12112            .unwrap();
12113        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12114
12115        hold.wait();
12116        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12117        let ack = module_rx.try_recv().unwrap().frame;
12118        assert_eq!(ack.header.ty, FrameType::HelloAck);
12119        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12120        assert_eq!(listed, vec![("aft".to_string(), true)]);
12121
12122        let (open_task, bind) = relay_route_open(
12123            &handler,
12124            client,
12125            &client_ctx.egress,
12126            &mut module_rx,
12127            303,
12128            "aft",
12129            "listed-after",
12130        )
12131        .await;
12132        assert!(handler
12133            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12134            .await
12135            .unwrap()
12136            .is_empty());
12137        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
12138        published_route(&client_rx.try_recv().unwrap().frame);
12139    }
12140
12141    /// A catalog watcher keys on the generation: when it changes, it re-reads
12142    /// the listing. So the generation must advance exactly when the module
12143    /// appears. If it advanced while the module was still hidden, a watcher
12144    /// would read the new generation without the module, and the module would
12145    /// later appear under that same generation with nothing telling the
12146    /// watcher to look again.
12147    #[tokio::test]
12148    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
12149        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
12150        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12151        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12152
12153        hold.wait();
12154        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
12155        assert!(listed.is_empty());
12156        assert_eq!(
12157            during, before,
12158            "the generation moved while the module was not listed"
12159        );
12160
12161        hold.wait();
12162        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12163        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12164        assert_eq!(listed, vec![("aft".to_string(), true)]);
12165        assert_eq!(
12166            after,
12167            before + 1,
12168            "the module appeared without exactly one generation bump"
12169        );
12170    }
12171
12172    /// A HELLO whose forwarding endpoint cannot be installed (here the
12173    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
12174    /// refused and leaves nothing behind: no registry entry, no catalog entry,
12175    /// and the catalog generation where it was.
12176    #[tokio::test]
12177    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
12178        let registry = Arc::new(Registry::default());
12179        let handler = ControlHandler::with_forwarding(
12180            Arc::clone(&registry),
12181            Arc::new(ForwardingTable::default()),
12182        );
12183        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12184        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12185        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12186        drop(module_rx);
12187
12188        let replies = handler
12189            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12190            .await
12191            .unwrap();
12192        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12193        assert!(registry.get_module("aft").unwrap().is_none());
12194        assert!(registry
12195            .get_module_by_connection(module_ctx.connection_id)
12196            .unwrap()
12197            .is_none());
12198        assert_eq!(
12199            catalog_snapshot(&handler, &client_ctx, 301).await,
12200            (before, Vec::new())
12201        );
12202    }
12203
12204    /// A handler whose `route.open` parks after queueing the route.bind and
12205    /// before marking it enqueued, with module "aft" registered on connection
12206    /// 40. Returns the two-party barrier that parks and releases it.
12207    async fn route_open_held_after_bind_send() -> (
12208        ControlHandler,
12209        Arc<tokio::sync::Barrier>,
12210        RouteCtx,
12211        mpsc::Receiver<crate::router::OutboundFrame>,
12212    ) {
12213        let hold = Arc::new(tokio::sync::Barrier::new(2));
12214        let handler = ControlHandler::with_forwarding(
12215            Arc::new(Registry::default()),
12216            Arc::new(ForwardingTable::default()),
12217        )
12218        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12219        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12220        hello_via_sink(
12221            &handler,
12222            &module_ctx,
12223            &mut module_rx,
12224            hello_frame("aft", PROTOCOL_VERSION, 7),
12225        )
12226        .await;
12227        (handler, hold, module_ctx, module_rx)
12228    }
12229
12230    /// A fast module acks the route.bind, and the reader commits the route,
12231    /// before `route.open` has marked the relay enqueued. The daemon used to
12232    /// read the missing relay as an abandoned bind and send the module a
12233    /// GOODBYE for the route it had just committed: the client was told its
12234    /// route was open, the module unbound the channel, and the client's first
12235    /// request was dropped on the module side as unbound.
12236    #[tokio::test]
12237    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12238        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12239        let client = ConnectionId::new(41);
12240        let (client_ctx, mut client_rx) = route_ctx(client);
12241
12242        let (open_task, bind) = relay_route_open(
12243            &handler,
12244            client,
12245            &client_ctx.egress,
12246            &mut module_rx,
12247            200,
12248            "aft",
12249            "acked-before-mark",
12250        )
12251        .await;
12252        let (module_channel, module_epoch) = route_bind_channel(&bind);
12253        // route.open is now parked between queueing the bind and marking it.
12254        hold.wait().await;
12255
12256        // The module answers inside that window, and the route commits.
12257        assert!(handler
12258            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12259            .await
12260            .unwrap()
12261            .is_empty());
12262        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12263
12264        hold.wait().await;
12265        assert!(
12266            open_task.await.unwrap().is_empty(),
12267            "route.open answers accepted"
12268        );
12269
12270        // Every frame route.open could have sent is already queued, because it
12271        // has returned; the module must have been sent nothing.
12272        match module_rx.try_recv() {
12273            Err(mpsc::error::TryRecvError::Empty) => {}
12274            Ok(outbound) => panic!(
12275                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12276                outbound.frame.header.ty,
12277                outbound.frame.header.channel,
12278                outbound.frame.header.epoch
12279            ),
12280            Err(err) => panic!("module egress closed: {err}"),
12281        }
12282
12283        // The client's first request on the route is the next thing the module
12284        // reads, on the binding it just created.
12285        let router = Router::with_control_handler(Arc::new(handler.clone()));
12286        let request = Frame::build(
12287            FrameType::Request,
12288            Flags::new(true, Priority::Interactive, false),
12289            client_channel,
12290            client_epoch,
12291            201,
12292            b"first request".to_vec(),
12293        )
12294        .unwrap();
12295        router
12296            .route_for_connection(&client_ctx, request)
12297            .await
12298            .unwrap();
12299        let delivered = module_rx.try_recv().unwrap().frame;
12300        assert_eq!(delivered.header.ty, FrameType::Request);
12301        assert_eq!(
12302            (delivered.header.channel, delivered.header.epoch),
12303            (module_channel, module_epoch)
12304        );
12305        assert_eq!(delivered.body, b"first request");
12306    }
12307
12308    /// The client goes away while `route.open` is still queueing the bind, so
12309    /// the relay is dropped with no module reply. The teardown cannot send the
12310    /// GOODBYE itself (it could reach the module ahead of the bind), so
12311    /// `route.open` must send it once the bind is queued; otherwise the module
12312    /// keeps a binding the daemon will never commit.
12313    #[tokio::test]
12314    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12315        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12316        let client = ConnectionId::new(42);
12317        let (client_ctx, _client_rx) = route_ctx(client);
12318
12319        let (open_task, bind) = relay_route_open(
12320            &handler,
12321            client,
12322            &client_ctx.egress,
12323            &mut module_rx,
12324            300,
12325            "aft",
12326            "released-before-mark",
12327        )
12328        .await;
12329        let (module_channel, module_epoch) = route_bind_channel(&bind);
12330        hold.wait().await;
12331
12332        handler.cleanup_connection(client).unwrap();
12333        assert!(
12334            module_rx.try_recv().is_err(),
12335            "client teardown must not GOODBYE a bind route.open is still queueing"
12336        );
12337
12338        hold.wait().await;
12339        let response = open_task.await.unwrap();
12340        assert_eq!(response.len(), 1);
12341        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12342
12343        let goodbye = module_rx
12344            .try_recv()
12345            .expect("module is told to drop the binding the daemon abandoned")
12346            .frame;
12347        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12348        assert_eq!(
12349            (goodbye.header.channel, goodbye.header.epoch),
12350            (module_channel, module_epoch)
12351        );
12352        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12353    }
12354
12355    /// The module acks inside the same window, but the client's egress closed
12356    /// first, so the reader refuses to commit. The ack proves the module holds
12357    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
12358    /// send a second one.
12359    #[tokio::test]
12360    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12361        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12362        let client = ConnectionId::new(43);
12363        let (client_ctx, client_rx) = route_ctx(client);
12364
12365        let (open_task, bind) = relay_route_open(
12366            &handler,
12367            client,
12368            &client_ctx.egress,
12369            &mut module_rx,
12370            400,
12371            "aft",
12372            "gone-client-before-mark",
12373        )
12374        .await;
12375        let (module_channel, module_epoch) = route_bind_channel(&bind);
12376        hold.wait().await;
12377
12378        drop(client_rx);
12379        assert!(client_ctx.egress.is_closed());
12380        handler
12381            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12382            .await
12383            .unwrap();
12384
12385        hold.wait().await;
12386        let response = open_task.await.unwrap();
12387        assert_eq!(response.len(), 1);
12388        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12389
12390        let goodbye = module_rx
12391            .try_recv()
12392            .expect("module is told to drop the binding it acked")
12393            .frame;
12394        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12395        assert_eq!(
12396            (goodbye.header.channel, goodbye.header.epoch),
12397            (module_channel, module_epoch)
12398        );
12399        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12400        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12401    }
12402
12403    /// The fence at the module-loop boundary, stated as its own contract: which
12404    /// forwarding failures are allowed to end the module connection that is being
12405    /// served. A `ConnectionClosing` naming some client is about that client, and
12406    /// a module connection is shared; the same error naming the module's own
12407    /// connection is about this connection and must stay fatal, as must failures
12408    /// that are about the forwarding table itself.
12409    #[test]
12410    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12411        let handler = ControlHandler::default();
12412        let module_connection = ConnectionId::new(30);
12413        let client_connection = ConnectionId::new(31);
12414
12415        handler
12416            .refuse_to_end_module_connection_for_a_client(
12417                module_connection,
12418                77,
12419                ForwardingError::ConnectionClosing {
12420                    connection_id: client_connection,
12421                },
12422            )
12423            .expect("a closing client must never end the module connection");
12424
12425        assert!(matches!(
12426            handler.refuse_to_end_module_connection_for_a_client(
12427                module_connection,
12428                78,
12429                ForwardingError::ConnectionClosing {
12430                    connection_id: module_connection,
12431                },
12432            ),
12433            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12434                connection_id
12435            })) if connection_id == module_connection
12436        ));
12437        assert!(matches!(
12438            handler.refuse_to_end_module_connection_for_a_client(
12439                module_connection,
12440                79,
12441                ForwardingError::Poisoned,
12442            ),
12443            Err(RouterError::Forwarding(ForwardingError::Poisoned))
12444        ));
12445        assert!(matches!(
12446            handler.refuse_to_end_module_connection_for_a_client(
12447                module_connection,
12448                80,
12449                ForwardingError::StaleModuleEndpoint,
12450            ),
12451            Err(RouterError::Forwarding(
12452                ForwardingError::StaleModuleEndpoint
12453            ))
12454        ));
12455    }
12456
12457    /// The spawn-attestation guard is what stops a connected module from claiming
12458    /// another module's identity and being stamped `Reserved` for it. Every other
12459    /// test that supplies a consumer_identity supplies a CORRECT one, because a
12460    /// correct one is what the rest of the flow needs -- so the guard's rejection
12461    /// branch was never the subject of an assertion, only its acceptance branch.
12462    ///
12463    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
12464    /// whole subc-core library suite green; only the forwarding integration tests
12465    /// notice, and they notice for unrelated reasons. This test exists so the
12466    /// refusal itself is asserted where the guard lives: it fails if the identity
12467    /// check stops refusing, which is the direction that matters, since a guard
12468    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
12469    #[tokio::test]
12470    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12471        let registry = Arc::new(Registry::default());
12472        let forwarding = Arc::new(ForwardingTable::default());
12473        let supervisor = SupervisorHandle::new();
12474        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12475        let handler =
12476            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12477                .with_supervisor(supervisor);
12478
12479        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12480        hello_via_sink(
12481            &handler,
12482            &target_ctx,
12483            &mut target_rx,
12484            hello_frame("target", PROTOCOL_VERSION, 1),
12485        )
12486        .await;
12487
12488        // A real supervised module id presenting the wrong nonce. This is the
12489        // impersonation case: the attacker knows a privileged module_id, which is
12490        // public, and guesses at the nonce, which is not.
12491        let wrong_nonce = handler
12492            .handle_control_frame(
12493                &route_ctx(ConnectionId::new(91)).0,
12494                route_open_frame_with_admission_facts(
12495                    20,
12496                    "target",
12497                    unique_project_root("admission-facts"),
12498                    Some(subc_control::ConsumerIdentity {
12499                        module_id: "fed".to_string(),
12500                        launch_nonce: "not-the-real-nonce".to_string(),
12501                    }),
12502                    None,
12503                ),
12504            )
12505            .await
12506            .unwrap();
12507        assert_eq!(
12508            parse_error(&wrong_nonce[0])["code"],
12509            "bad_consumer_identity",
12510            "a mismatched launch nonce must be refused, not stamped Reserved"
12511        );
12512
12513        // A module id the supervisor never spawned at all, so no nonce exists to
12514        // compare against. An implementation that treats "no record" as "nothing
12515        // to check" fails open here while passing the case above.
12516        let never_spawned = handler
12517            .handle_control_frame(
12518                &route_ctx(ConnectionId::new(92)).0,
12519                route_open_frame_with_admission_facts(
12520                    21,
12521                    "target",
12522                    unique_project_root("admission-facts"),
12523                    Some(subc_control::ConsumerIdentity {
12524                        module_id: "never-spawned".to_string(),
12525                        launch_nonce: "any-nonce".to_string(),
12526                    }),
12527                    None,
12528                ),
12529            )
12530            .await
12531            .unwrap();
12532        assert_eq!(
12533            parse_error(&never_spawned[0])["code"],
12534            "bad_consumer_identity",
12535            "an unspawned module_id must be refused rather than accepted for lack of a record"
12536        );
12537    }
12538
12539    /// The refusal test above proves the guard says NO. Nothing proved it can say
12540    /// YES, and the difference is not academic: replacing the whole authorization
12541    /// with `false` -- admitting no consumer identity at all, revoking Reserved
12542    /// standing for every supervised module in the fleet -- leaves 110 of the 111
12543    /// library tests GREEN. The one that notices does so by HANGING, because it
12544    /// waits for a bind that can no longer happen.
12545    ///
12546    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
12547    /// or flaky test, invites a RETRY rather than an investigation, and the retry
12548    /// hangs too and gets blamed on the runner. So a total revocation of the
12549    /// daemon's trust grant would have shipped behind a symptom nobody attributes
12550    /// to code.
12551    ///
12552    /// The bias is structural rather than accidental. A REFUSAL looks like a
12553    /// failure someone writes a test for; a GRANT looks like the happy path. Every
12554    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
12555    /// suite for that reason, and this one is the purest case in the daemon.
12556    ///
12557    /// This test asserts the EFFECT rather than the absence of an error: the module
12558    /// receives a RouteBind and it carries `Reserved` naming the attested module.
12559    /// A guard that admitted nobody would produce no bind at all; one that admitted
12560    /// everybody would stamp the wrong principal, which the refusal test catches.
12561    #[tokio::test]
12562    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12563        let registry = Arc::new(Registry::default());
12564        let forwarding = Arc::new(ForwardingTable::default());
12565        let supervisor = SupervisorHandle::new();
12566        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12567        let handler =
12568            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12569                .with_supervisor(supervisor);
12570
12571        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12572        hello_via_sink(
12573            &handler,
12574            &target_ctx,
12575            &mut target_rx,
12576            hello_frame("target", PROTOCOL_VERSION, 1),
12577        )
12578        .await;
12579
12580        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12581        let route_handler = handler.clone();
12582        let route_task = tokio::spawn(async move {
12583            route_handler
12584                .handle_control_frame(
12585                    &client_ctx,
12586                    route_open_frame_with_admission_facts(
12587                        30,
12588                        "target",
12589                        unique_project_root("admission-facts"),
12590                        Some(subc_control::ConsumerIdentity {
12591                            module_id: "fed".to_string(),
12592                            launch_nonce: "fed-nonce".to_string(),
12593                        }),
12594                        None,
12595                    ),
12596                )
12597                .await
12598                .unwrap()
12599        });
12600
12601        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
12602        // the very mutation it exists to catch -- a guard that admits nobody -- no
12603        // bind is ever sent, so it HUNG rather than failing. That reproduces the
12604        // exact defect being fixed: a total revocation detected only as a stalled
12605        // suite, which reads as flakiness and invites a retry. An acceptance test
12606        // that waits for an effect must bound the wait, or a red becomes a hang.
12607        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12608            .await
12609            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12610            .expect("module control channel closed before route.bind");
12611        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12612        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12613            panic!("expected route.bind")
12614        };
12615        assert_eq!(
12616            principal,
12617            Some(Principal::Reserved {
12618                module_id: "fed".to_string()
12619            }),
12620            "a correctly attested consumer must be stamped Reserved for its own id"
12621        );
12622
12623        handler
12624            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12625            .await
12626            .unwrap();
12627        assert!(route_task.await.unwrap().is_empty());
12628        assert!(
12629            matches!(
12630                serde_json::from_slice::<ClientControlResponse>(
12631                    &client_rx.recv().await.unwrap().body
12632                )
12633                .unwrap(),
12634                ClientControlResponse::RouteOpen { .. }
12635            ),
12636            "the route must actually open, not merely avoid an error"
12637        );
12638    }
12639
12640    #[tokio::test(start_paused = true)]
12641    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12642        let registry = Arc::new(Registry::default());
12643        let forwarding = Arc::new(ForwardingTable::default());
12644        let supervisor = SupervisorHandle::new();
12645        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12646        let handler =
12647            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12648                .with_supervisor(supervisor);
12649
12650        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12651        hello_via_sink(
12652            &handler,
12653            &target_ctx,
12654            &mut target_rx,
12655            hello_frame("target", PROTOCOL_VERSION, 1),
12656        )
12657        .await;
12658
12659        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12660        let direct_handler = handler.clone();
12661        let direct_open = tokio::spawn(async move {
12662            direct_handler
12663                .handle_control_frame(
12664                    &direct_ctx,
12665                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
12666                )
12667                .await
12668                .unwrap()
12669        });
12670        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12671            .await
12672            .expect("no direct route.bind within 5s")
12673            .expect("target control channel closed before direct route.bind");
12674        handler
12675            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12676            .await
12677            .unwrap();
12678        assert!(direct_open.await.unwrap().is_empty());
12679        let _ = direct_rx.recv().await.unwrap();
12680
12681        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12682        let reserved_handler = handler.clone();
12683        let reserved_open = tokio::spawn(async move {
12684            reserved_handler
12685                .handle_control_frame(
12686                    &reserved_ctx,
12687                    route_open_frame_with_admission_facts(
12688                        3,
12689                        "target",
12690                        unique_project_root("admission-facts"),
12691                        Some(ConsumerIdentity {
12692                            module_id: "fed".to_string(),
12693                            launch_nonce: "fed-nonce".to_string(),
12694                        }),
12695                        None,
12696                    ),
12697                )
12698                .await
12699                .unwrap()
12700        });
12701        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12702            .await
12703            .expect("no reserved route.bind within 5s")
12704            .expect("target control channel closed before reserved route.bind");
12705        handler
12706            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12707            .await
12708            .unwrap();
12709        assert!(reserved_open.await.unwrap().is_empty());
12710        let _ = reserved_rx.recv().await.unwrap();
12711
12712        forwarding
12713            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12714            .unwrap();
12715        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12716        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12717            module_id: Some("target".to_string()),
12718        })
12719        .unwrap();
12720        let census_frame =
12721            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12722        let response = handler
12723            .handle_control_frame(&census_ctx, census_frame)
12724            .await
12725            .unwrap()
12726            .pop()
12727            .unwrap();
12728        let actual: Value = serde_json::from_slice(&response.body).unwrap();
12729        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12730        assert!(matches!(
12731            decoded,
12732            ClientControlResponse::SupervisorRoutes { .. }
12733        ));
12734        let routes = actual["modules"][0]["routes"].as_array().unwrap();
12735        assert_eq!(routes.len(), 2);
12736        assert!(routes.iter().all(|route| route["draining"] == true));
12737        // The census carries WHY: the reason the drain was begun with, in the
12738        // route.closing vocabulary, on every draining route this drain marked.
12739        assert!(
12740            routes.iter().all(|route| route["drain_reason"] == "reload"),
12741            "draining routes must name the drain's reason: {routes:?}"
12742        );
12743        assert!(routes.iter().any(|route| {
12744            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12745        }));
12746        assert!(routes.iter().any(|route| {
12747            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12748        }));
12749
12750        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12751            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12752        if std::env::var_os("UPDATE_GOLDEN").is_some() {
12753            std::fs::write(
12754                &golden_path,
12755                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12756            )
12757            .unwrap();
12758        }
12759        let expected: Value =
12760            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12761        assert_eq!(actual, expected);
12762    }
12763
12764    async fn query_live_roots(
12765        handler: &ControlHandler,
12766        module_ctx: &RouteCtx,
12767    ) -> ModuleControlResponseToModule {
12768        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12769        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12770        let response = handler
12771            .handle_control_frame(module_ctx, frame)
12772            .await
12773            .unwrap()
12774            .pop()
12775            .unwrap();
12776        serde_json::from_slice(&response.body).unwrap()
12777    }
12778
12779    #[tokio::test(start_paused = true)]
12780    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12781        let registry = Arc::new(Registry::default());
12782        let forwarding = Arc::new(ForwardingTable::default());
12783        let handler = ControlHandler::with_forwarding(registry, forwarding);
12784        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12785        hello_via_sink(
12786            &handler,
12787            &target_ctx,
12788            &mut target_rx,
12789            hello_frame("target", PROTOCOL_VERSION, 1),
12790        )
12791        .await;
12792        let root = unique_project_root("live-roots-known");
12793        let path = ProjectRootId::from_path_allowing_missing(root.path())
12794            .unwrap()
12795            .as_path()
12796            .to_path_buf();
12797        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12798        let open_handler = handler.clone();
12799        let opened = tokio::spawn(async move {
12800            open_handler
12801                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12802                .await
12803                .unwrap()
12804        });
12805        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12806            .await
12807            .unwrap()
12808            .unwrap();
12809        handler
12810            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12811            .await
12812            .unwrap();
12813        assert!(opened.await.unwrap().is_empty());
12814        let _ = client_rx.recv().await.unwrap();
12815
12816        let root = unique_project_root("live-roots-pending");
12817        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12818            .unwrap()
12819            .as_path()
12820            .to_path_buf();
12821        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12822        let open_handler = handler.clone();
12823        let pending = tokio::spawn(async move {
12824            open_handler
12825                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12826                .await
12827                .unwrap()
12828        });
12829        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12830            .await
12831            .unwrap()
12832            .unwrap();
12833        let actual = query_live_roots(&handler, &target_ctx).await;
12834        let ModuleControlResponseToModule::LiveRoots {
12835            roots,
12836            unknown_root_bindings,
12837            total_bindings,
12838        } = actual
12839        else {
12840            panic!("expected live roots")
12841        };
12842        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12843        assert_eq!(unknown_root_bindings, 0);
12844        assert_eq!(
12845            roots.len(),
12846            2,
12847            "root-known arm must retain each canonical root"
12848        );
12849        assert_eq!(
12850            total_bindings,
12851            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12852        );
12853        let counts = roots
12854            .iter()
12855            .map(|root| (root.project_root.clone(), root.bound, root.pending))
12856            .collect::<Vec<_>>();
12857        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12858        expected.sort_by(|a, b| a.0.cmp(&b.0));
12859        assert_eq!(
12860            counts, expected,
12861            "roots must sort by path and count pending separately"
12862        );
12863        handler
12864            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12865            .await
12866            .unwrap();
12867        assert!(pending.await.unwrap().is_empty());
12868    }
12869
12870    #[tokio::test(start_paused = true)]
12871    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12872        let forwarding = Arc::new(ForwardingTable::default());
12873        let handler =
12874            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12875        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12876        hello_via_sink(
12877            &handler,
12878            &target_ctx,
12879            &mut target_rx,
12880            hello_frame("target", PROTOCOL_VERSION, 1),
12881        )
12882        .await;
12883        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12884        let pending = forwarding
12885            .begin_route_bind_relay_for_test(
12886                client_ctx.connection_id,
12887                client_ctx.egress.clone(),
12888                2,
12889                "target",
12890            )
12891            .unwrap();
12892        forwarding
12893            .complete_pending_relay(
12894                target_ctx.connection_id,
12895                pending.corr,
12896                RouteBindRelayOutcome::Accepted,
12897            )
12898            .unwrap();
12899        let actual = query_live_roots(&handler, &target_ctx).await;
12900        let ModuleControlResponseToModule::LiveRoots {
12901            roots,
12902            unknown_root_bindings,
12903            total_bindings,
12904        } = actual
12905        else {
12906            panic!("expected live roots")
12907        };
12908        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12909        assert_eq!(
12910            unknown_root_bindings, 1,
12911            "unknown-root arm must not read as no bindings"
12912        );
12913        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12914        assert_eq!(
12915            total_bindings,
12916            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12917        );
12918    }
12919
12920    /// A module reads HELLO_ACK as its first frame and exits on anything else,
12921    /// so the ack has to be on its outbound queue before the module is
12922    /// routable. The connection loop writes a handler's replies only after the
12923    /// handler returns; this test stops in exactly that gap, runs a real
12924    /// route.open from another connection, and only then writes whatever the
12925    /// HELLO handler returned, the way the loop would. If the ack were still a
12926    /// reply, the route.bind request would reach the module first.
12927    #[tokio::test(start_paused = true)]
12928    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12929        let forwarding = Arc::new(ForwardingTable::default());
12930        let handler =
12931            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12932        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12933        let replies = handler
12934            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12935            .await
12936            .unwrap();
12937        let queued_by_hello = module_rx.len();
12938
12939        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12940        let open_handler = handler.clone();
12941        let open = tokio::spawn(async move {
12942            open_handler
12943                .handle_control_frame(
12944                    &client_ctx,
12945                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12946                )
12947                .await
12948                .unwrap()
12949        });
12950        // Let the route.open run until its route.bind is on the module's queue.
12951        let mut spins = 0;
12952        while module_rx.len() == queued_by_hello {
12953            spins += 1;
12954            assert!(spins < 10_000, "route.open never queued a route.bind");
12955            tokio::task::yield_now().await;
12956        }
12957
12958        // Now the connection loop's half: write the HELLO handler's replies.
12959        for reply in replies {
12960            module_ctx.egress.send(reply).await.unwrap();
12961        }
12962
12963        let first = module_rx.recv().await.unwrap().frame;
12964        assert_eq!(
12965            first.header.ty,
12966            FrameType::HelloAck,
12967            "the first frame a registering module reads must be its HELLO_ACK"
12968        );
12969        assert_eq!(first.header.corr, 7);
12970        let second = module_rx.recv().await.unwrap().frame;
12971        assert_eq!(second.header.ty, FrameType::Request);
12972        assert!(
12973            matches!(
12974                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12975                ModuleControlRequest::RouteBind { .. }
12976            ),
12977            "the route.bind follows the ack"
12978        );
12979        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12980
12981        handler
12982            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12983            .await
12984            .unwrap();
12985        assert!(open.await.unwrap().is_empty());
12986        let _ = client_rx.recv().await.unwrap();
12987    }
12988
12989    #[tokio::test(start_paused = true)]
12990    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12991        let handler = ControlHandler::with_forwarding(
12992            Arc::new(Registry::default()),
12993            Arc::new(ForwardingTable::default()),
12994        );
12995        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12996        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12997        hello_via_sink(
12998            &handler,
12999            &first_ctx,
13000            &mut first_rx,
13001            hello_frame("first", PROTOCOL_VERSION, 1),
13002        )
13003        .await;
13004        hello_via_sink(
13005            &handler,
13006            &second_ctx,
13007            &mut second_rx,
13008            hello_frame("second", PROTOCOL_VERSION, 2),
13009        )
13010        .await;
13011        let root = unique_project_root("second-only");
13012        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
13013        let cloned = handler.clone();
13014        let open = tokio::spawn(async move {
13015            cloned
13016                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
13017                .await
13018                .unwrap()
13019        });
13020        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
13021            .await
13022            .unwrap()
13023            .unwrap();
13024        let first = query_live_roots(&handler, &first_ctx).await;
13025        let second = query_live_roots(&handler, &second_ctx).await;
13026        assert!(
13027            matches!(
13028                first,
13029                ModuleControlResponseToModule::LiveRoots {
13030                    total_bindings: 0,
13031                    ..
13032                }
13033            ),
13034            "cross-module scope must not expose another module's roots"
13035        );
13036        assert!(
13037            matches!(
13038                second,
13039                ModuleControlResponseToModule::LiveRoots {
13040                    total_bindings: 1,
13041                    ..
13042                }
13043            ),
13044            "second module must see its pending route"
13045        );
13046        handler
13047            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
13048            .await
13049            .unwrap();
13050        assert!(open.await.unwrap().is_empty());
13051    }
13052
13053    #[tokio::test(start_paused = true)]
13054    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
13055        let handler = ControlHandler::with_forwarding(
13056            Arc::new(Registry::default()),
13057            Arc::new(ForwardingTable::default()),
13058        );
13059        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
13060        hello_via_sink(
13061            &handler,
13062            &target_ctx,
13063            &mut target_rx,
13064            hello_frame("target", PROTOCOL_VERSION, 1),
13065        )
13066        .await;
13067        let actual = query_live_roots(&handler, &target_ctx).await;
13068        let ModuleControlResponseToModule::LiveRoots {
13069            roots,
13070            unknown_root_bindings,
13071            total_bindings,
13072        } = actual
13073        else {
13074            panic!("expected live roots")
13075        };
13076        assert!(roots.is_empty());
13077        assert_eq!(unknown_root_bindings, 0);
13078        assert_eq!(total_bindings, 0);
13079        assert_eq!(
13080            total_bindings,
13081            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13082        );
13083    }
13084
13085    /// Read the vendored fed corpus rather than hand-building a package.
13086    ///
13087    /// A hand-built object encodes what the test author believed the carrier
13088    /// emits. These vectors are what it actually emits, and one of them exists
13089    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
13090    /// ignores additive unknown fields at the traversal emit terminus."
13091    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
13092        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
13093            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
13094        let text = std::fs::read_to_string(&path)
13095            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
13096        let vectors: Vec<(String, Value)> = text
13097            .lines()
13098            .filter(|line| !line.trim().is_empty())
13099            .map(|line| {
13100                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
13101                let id = entry["corpus_id"]
13102                    .as_str()
13103                    .expect("every vector carries a corpus_id")
13104                    .to_string();
13105                (id, entry["package"].clone())
13106            })
13107            .collect();
13108        // Pin the count: a corpus that silently shrinks would take its coverage
13109        // with it, and a suite reading N-1 vectors reports the same clean pass
13110        // as one reading N.
13111        assert_eq!(
13112            vectors.len(),
13113            3,
13114            "vendored fed corpus changed size; re-sync from subc-federation"
13115        );
13116
13117        // Pin what makes the corpus DISCRIMINATING, not just present.
13118        //
13119        // The relay test below takes its expected value from the corpus, so the
13120        // corpus supplies the test's power to detect a lossy relay rather than
13121        // its correctness. A relay that dropped unrecognised fields would still
13122        // be caught -- but only by a package carrying fields it does not know.
13123        // Shrink every package to the handful of keys any implementation would
13124        // recognise and the test keeps passing over an input that can no longer
13125        // fail, which is the same clean green as a corpus that shrank away.
13126        //
13127        // So assert the precondition rather than duplicating the packages here:
13128        // at least one vector must carry a field beyond the small common set.
13129        // That is one claim to maintain instead of nine, and it fails loudly if
13130        // a re-sync ever flattens the corpus.
13131        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
13132        let richest = vectors
13133            .iter()
13134            .filter_map(|(_, package)| package.as_object())
13135            .map(|object| {
13136                object
13137                    .keys()
13138                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
13139                    .count()
13140            })
13141            .max()
13142            .unwrap_or(0);
13143        assert!(
13144            richest >= 2,
13145            "vendored corpus no longer carries a package with unmodelled fields, \
13146             so the relay test can no longer distinguish a verbatim relay from a lossy one"
13147        );
13148
13149        vectors
13150    }
13151
13152    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
13153    ///
13154    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
13155    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
13156    /// three-key object, so a relay that quietly dropped fields it did not
13157    /// recognise would satisfy it. These vectors carry nine keys including ones
13158    /// this crate has no type for, so a typed relay fails here and only here.
13159    #[tokio::test]
13160    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
13161        for (corpus_id, package) in fed_admission_facts_vectors() {
13162            let registry = Arc::new(Registry::default());
13163            let forwarding = Arc::new(ForwardingTable::default());
13164            let supervisor = SupervisorHandle::new();
13165            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13166            let handler =
13167                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13168                    .with_supervisor(supervisor)
13169                    .with_admission_facts_config(
13170                        Some("fed".to_string()),
13171                        Some(vec!["target".to_string()]),
13172                    );
13173
13174            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
13175            hello_via_sink(
13176                &handler,
13177                &target_ctx,
13178                &mut target_rx,
13179                hello_frame("target", PROTOCOL_VERSION, 1),
13180            )
13181            .await;
13182
13183            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
13184            let route_handler = handler.clone();
13185            let expected = package.clone();
13186            let route_task = tokio::spawn(async move {
13187                route_handler
13188                    .handle_control_frame(
13189                        &client_ctx,
13190                        route_open_frame_with_admission_facts(
13191                            20,
13192                            "target",
13193                            unique_project_root("admission-facts"),
13194                            Some(subc_control::ConsumerIdentity {
13195                                module_id: "fed".to_string(),
13196                                launch_nonce: "fed-nonce".to_string(),
13197                            }),
13198                            Some(package),
13199                        ),
13200                    )
13201                    .await
13202                    .unwrap()
13203            });
13204
13205            let bind_frame = target_rx.recv().await.unwrap();
13206            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13207            let ModuleControlRequest::RouteBind {
13208                admission_facts, ..
13209            } = bind
13210            else {
13211                panic!("{corpus_id}: expected route.bind")
13212            };
13213            assert_eq!(
13214                admission_facts,
13215                Some(expected),
13216                "{corpus_id}: relay must not add, drop or reshape any field"
13217            );
13218
13219            handler
13220                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13221                .await
13222                .unwrap();
13223            route_task.await.unwrap();
13224        }
13225    }
13226
13227    #[tokio::test]
13228    async fn route_bind_never_relays_the_openers_build_class() {
13229        let registry = Arc::new(Registry::default());
13230        let forwarding = Arc::new(ForwardingTable::default());
13231        let handler =
13232            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13233        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13234        hello_via_sink(
13235            &handler,
13236            &target_ctx,
13237            &mut target_rx,
13238            hello_frame("target", PROTOCOL_VERSION, 1),
13239        )
13240        .await;
13241
13242        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13243        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13244        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13245            busy_refusals: None,
13246            target: RouteTarget::ToolProvider {
13247                module_id: "target".to_string(),
13248            },
13249            identity,
13250            consumer_identity: None,
13251            consumer_capabilities: None,
13252            role_versions: None,
13253            admission_facts: None,
13254            scope: None,
13255        })
13256        .unwrap();
13257        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13258
13259        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13260        let route_handler = handler.clone();
13261        let route_task = tokio::spawn(async move {
13262            route_handler
13263                .handle_control_frame(&client_ctx, open)
13264                .await
13265                .unwrap()
13266        });
13267        let bind_frame = target_rx.recv().await.unwrap();
13268        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13269        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13270            panic!("expected route.bind")
13271        };
13272        assert_eq!(
13273            identity.build_class, None,
13274            "route.bind carried the build class"
13275        );
13276        assert!(
13277            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13278            "the route.bind bytes still name build_class"
13279        );
13280        handler
13281            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13282            .await
13283            .unwrap();
13284        route_task.await.unwrap();
13285    }
13286
13287    #[tokio::test]
13288    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13289        let registry = Arc::new(Registry::default());
13290        let forwarding = Arc::new(ForwardingTable::default());
13291        let supervisor = SupervisorHandle::new();
13292        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13293        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13294        let handler =
13295            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13296                .with_supervisor(supervisor)
13297                .with_admission_facts_config(
13298                    Some("fed".to_string()),
13299                    Some(vec!["target".to_string()]),
13300                );
13301
13302        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13303        hello_via_sink(
13304            &handler,
13305            &target_ctx,
13306            &mut target_rx,
13307            hello_frame("target", PROTOCOL_VERSION, 1),
13308        )
13309        .await;
13310        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13311        hello_via_sink(
13312            &handler,
13313            &other_ctx,
13314            &mut other_rx,
13315            hello_frame("other", PROTOCOL_VERSION, 2),
13316        )
13317        .await;
13318
13319        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13320        let expected_facts = facts.clone();
13321        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13322        let route_handler = handler.clone();
13323        let route_task = tokio::spawn(async move {
13324            route_handler
13325                .handle_control_frame(
13326                    &client_ctx,
13327                    route_open_frame_with_admission_facts(
13328                        10,
13329                        "target",
13330                        unique_project_root("admission-facts"),
13331                        Some(subc_control::ConsumerIdentity {
13332                            module_id: "fed".to_string(),
13333                            launch_nonce: "fed-nonce".to_string(),
13334                        }),
13335                        Some(facts.clone()),
13336                    ),
13337                )
13338                .await
13339                .unwrap()
13340        });
13341        let bind_frame = target_rx.recv().await.unwrap();
13342        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13343        let ModuleControlRequest::RouteBind {
13344            admission_facts, ..
13345        } = bind
13346        else {
13347            panic!("expected route.bind")
13348        };
13349        assert_eq!(admission_facts, Some(expected_facts));
13350        handler
13351            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13352            .await
13353            .unwrap();
13354        assert!(route_task.await.unwrap().is_empty());
13355        assert!(matches!(
13356            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13357                .unwrap(),
13358            ClientControlResponse::RouteOpen { .. }
13359        ));
13360
13361        let direct = handler
13362            .handle_control_frame(
13363                &route_ctx(ConnectionId::new(73)).0,
13364                route_open_frame_with_admission_facts(
13365                    11,
13366                    "target",
13367                    unique_project_root("admission-facts"),
13368                    None,
13369                    Some(json!({"x": 1})),
13370                ),
13371            )
13372            .await
13373            .unwrap();
13374        assert_eq!(
13375            parse_error(&direct[0])["code"],
13376            "admission_facts_not_permitted"
13377        );
13378
13379        let different_reserved = handler
13380            .handle_control_frame(
13381                &route_ctx(ConnectionId::new(77)).0,
13382                route_open_frame_with_admission_facts(
13383                    15,
13384                    "target",
13385                    unique_project_root("admission-facts"),
13386                    Some(subc_control::ConsumerIdentity {
13387                        module_id: "other".to_string(),
13388                        launch_nonce: "other-nonce".to_string(),
13389                    }),
13390                    Some(json!({"x": 1})),
13391                ),
13392            )
13393            .await
13394            .unwrap();
13395        assert_eq!(
13396            parse_error(&different_reserved[0])["code"],
13397            "admission_facts_not_permitted"
13398        );
13399
13400        let other_target = handler
13401            .handle_control_frame(
13402                &route_ctx(ConnectionId::new(74)).0,
13403                route_open_frame_with_admission_facts(
13404                    12,
13405                    "other",
13406                    unique_project_root("admission-facts"),
13407                    Some(subc_control::ConsumerIdentity {
13408                        module_id: "fed".to_string(),
13409                        launch_nonce: "fed-nonce".to_string(),
13410                    }),
13411                    Some(json!({"x": 1})),
13412                ),
13413            )
13414            .await
13415            .unwrap();
13416        assert_eq!(
13417            parse_error(&other_target[0])["code"],
13418            "admission_facts_target_not_allowed"
13419        );
13420
13421        let nonexistent = handler
13422            .handle_control_frame(
13423                &route_ctx(ConnectionId::new(75)).0,
13424                route_open_frame_with_admission_facts(
13425                    13,
13426                    "missing",
13427                    unique_project_root("admission-facts"),
13428                    None,
13429                    Some(json!({"x": 1})),
13430                ),
13431            )
13432            .await
13433            .unwrap();
13434        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13435
13436        let described = handler
13437            .handle_control_frame(
13438                &route_ctx(ConnectionId::new(76)).0,
13439                Frame::build(
13440                    FrameType::Request,
13441                    control_flags(),
13442                    0,
13443                    0,
13444                    14,
13445                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13446                )
13447                .unwrap(),
13448            )
13449            .await
13450            .unwrap();
13451        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13452            serde_json::from_slice(&described[0].body).unwrap()
13453        else {
13454            panic!("expected server.describe response")
13455        };
13456        assert!(capabilities
13457            .iter()
13458            .any(|cap| cap == "admission_facts_relay_v1"));
13459    }
13460
13461    #[tokio::test]
13462    async fn admission_facts_without_configured_carrier_are_rejected() {
13463        let registry = Arc::new(Registry::default());
13464        let forwarding = Arc::new(ForwardingTable::default());
13465        let handler = ControlHandler::with_forwarding(registry, forwarding);
13466        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13467        hello_via_sink(
13468            &handler,
13469            &target_ctx,
13470            &mut target_rx,
13471            hello_frame("target", PROTOCOL_VERSION, 1),
13472        )
13473        .await;
13474
13475        let responses = handler
13476            .handle_control_frame(
13477                &route_ctx(ConnectionId::new(79)).0,
13478                route_open_frame_with_admission_facts(
13479                    16,
13480                    "target",
13481                    unique_project_root("admission-facts"),
13482                    None,
13483                    Some(json!({"x": 1})),
13484                ),
13485            )
13486            .await
13487            .unwrap();
13488        assert_eq!(
13489            parse_error(&responses[0])["code"],
13490            "admission_facts_not_permitted"
13491        );
13492    }
13493
13494    #[tokio::test]
13495    async fn route_open_relays_consumer_capabilities_verbatim() {
13496        let registry = Arc::new(Registry::default());
13497        let forwarding = Arc::new(ForwardingTable::default());
13498        let handler =
13499            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13500        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13501        hello_via_sink(
13502            &handler,
13503            &module_ctx,
13504            &mut module_rx,
13505            hello_frame("aft", PROTOCOL_VERSION, 7),
13506        )
13507        .await;
13508
13509        let expected = vec!["elicitation".to_string(), "roots".to_string()];
13510        let expected_for_request = expected.clone();
13511        let project_root = unique_project_root("consumer-capabilities-present");
13512        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13513        let route_handler = handler.clone();
13514        let route_task = tokio::spawn(async move {
13515            route_handler
13516                .handle_control_frame(
13517                    &client_ctx,
13518                    route_open_frame_with_consumer_capabilities(
13519                        401,
13520                        "aft",
13521                        project_root,
13522                        Some(expected_for_request),
13523                    ),
13524                )
13525                .await
13526                .unwrap()
13527        });
13528        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13529            .await
13530            .unwrap()
13531            .unwrap();
13532        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13533        let ModuleControlRequest::RouteBind {
13534            consumer_capabilities,
13535            ..
13536        } = bind
13537        else {
13538            panic!("expected route.bind request, got {bind:?}");
13539        };
13540        assert_eq!(consumer_capabilities, Some(expected.clone()));
13541
13542        handler
13543            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13544            .await
13545            .unwrap();
13546        let route_response = route_task.await.unwrap();
13547        assert!(route_response.is_empty());
13548        let published = client_rx.recv().await.unwrap();
13549        assert!(matches!(
13550            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13551            ClientControlResponse::RouteOpen { .. }
13552        ));
13553    }
13554
13555    #[tokio::test]
13556    async fn route_open_without_consumer_capabilities_relays_none() {
13557        let registry = Arc::new(Registry::default());
13558        let forwarding = Arc::new(ForwardingTable::default());
13559        let handler =
13560            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13561        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13562        hello_via_sink(
13563            &handler,
13564            &module_ctx,
13565            &mut module_rx,
13566            hello_frame("aft", PROTOCOL_VERSION, 7),
13567        )
13568        .await;
13569
13570        let project_root = unique_project_root("consumer-capabilities-absent");
13571        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13572        let route_handler = handler.clone();
13573        let route_task = tokio::spawn(async move {
13574            route_handler
13575                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13576                .await
13577                .unwrap()
13578        });
13579        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13580            .await
13581            .unwrap()
13582            .unwrap();
13583        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13584        let ModuleControlRequest::RouteBind {
13585            consumer_capabilities,
13586            ..
13587        } = bind
13588        else {
13589            panic!("expected route.bind request, got {bind:?}");
13590        };
13591        assert_eq!(consumer_capabilities, None);
13592
13593        handler
13594            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13595            .await
13596            .unwrap();
13597        let route_response = route_task.await.unwrap();
13598        assert!(route_response.is_empty());
13599        let published = client_rx.recv().await.unwrap();
13600        assert!(matches!(
13601            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13602            ClientControlResponse::RouteOpen { .. }
13603        ));
13604    }
13605
13606    /// Opens a route to a freshly registered `aft` with `sent` as the
13607    /// route.open's role_versions, acks the bind, and returns the role_versions
13608    /// the module's bind carried.
13609    async fn bind_role_versions_for(
13610        sent: Option<BTreeMap<String, String>>,
13611        connection: u64,
13612    ) -> Option<BTreeMap<String, String>> {
13613        let registry = Arc::new(Registry::default());
13614        let forwarding = Arc::new(ForwardingTable::default());
13615        let handler =
13616            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13617        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13618        hello_via_sink(
13619            &handler,
13620            &module_ctx,
13621            &mut module_rx,
13622            hello_frame("aft", PROTOCOL_VERSION, 7),
13623        )
13624        .await;
13625        let project_root = unique_project_root("role-versions");
13626        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13627        let route_handler = handler.clone();
13628        let route_task = tokio::spawn(async move {
13629            route_handler
13630                .handle_control_frame(
13631                    &client_ctx,
13632                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
13633                )
13634                .await
13635                .unwrap()
13636        });
13637        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13638            .await
13639            .expect("a well-formed route.open reaches the module as a bind")
13640            .unwrap();
13641        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13642        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13643            panic!("expected route.bind request, got {bind:?}");
13644        };
13645        handler
13646            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13647            .await
13648            .unwrap();
13649        assert!(route_task.await.unwrap().is_empty());
13650        let published = client_rx.recv().await.unwrap();
13651        assert!(matches!(
13652            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13653            ClientControlResponse::RouteOpen { .. }
13654        ));
13655        role_versions
13656    }
13657
13658    #[tokio::test]
13659    async fn route_open_relays_role_versions_verbatim() {
13660        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13661        assert_eq!(
13662            bind_role_versions_for(Some(sent.clone()), 141).await,
13663            Some(sent)
13664        );
13665    }
13666
13667    /// An empty map declares nothing, so the provider sees no field rather
13668    /// than an empty object it would have to treat as a second "none".
13669    #[tokio::test]
13670    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13671        assert_eq!(bind_role_versions_for(None, 143).await, None);
13672        assert_eq!(
13673            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13674            None
13675        );
13676    }
13677
13678    #[tokio::test]
13679    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13680        let registry = Arc::new(Registry::default());
13681        let forwarding = Arc::new(ForwardingTable::default());
13682        let handler =
13683            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13684        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13685        hello_via_sink(
13686            &handler,
13687            &module_ctx,
13688            &mut module_rx,
13689            hello_frame("aft", PROTOCOL_VERSION, 7),
13690        )
13691        .await;
13692
13693        let nine: BTreeMap<String, String> = (0..9)
13694            .map(|index| (format!("role-{index}"), "v1".to_string()))
13695            .collect();
13696        for (label, malformed) in [
13697            (
13698                "invalid role name",
13699                role_versions(&[("Tool_Provider", "v1")]),
13700            ),
13701            ("invalid version", role_versions(&[("tool-provider", "v0")])),
13702            ("nine entries", nine),
13703        ] {
13704            // A refused open answers at once; one that reached the module
13705            // would wait for its bind ack and trip this timeout.
13706            let responses = tokio::time::timeout(
13707                Duration::from_secs(1),
13708                handler.handle_control_frame(
13709                    &route_ctx(ConnectionId::new(148)).0,
13710                    route_open_frame_with_role_versions(
13711                        404,
13712                        "aft",
13713                        unique_project_root("role-versions-malformed"),
13714                        Some(malformed),
13715                    ),
13716                ),
13717            )
13718            .await
13719            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13720            .unwrap();
13721            assert_eq!(responses.len(), 1, "{label}");
13722            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13723            let error = parse_error(&responses[0]);
13724            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13725            assert_eq!(
13726                error["detail"]["field"], "role_versions",
13727                "{label}: {error}"
13728            );
13729            assert!(
13730                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13731                "{label}: a malformed declaration is terminal"
13732            );
13733            assert!(
13734                module_rx.try_recv().is_err(),
13735                "{label}: the module must never see a bind"
13736            );
13737        }
13738    }
13739
13740    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
13741    /// consumer can tell this daemon forwards the field from one that would
13742    /// drop it.
13743    #[tokio::test]
13744    async fn route_role_versions_capability_is_advertised() {
13745        let handler = ControlHandler::new(Arc::new(Registry::default()));
13746        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13747        let ack = hello_via_sink(
13748            &handler,
13749            &ctx,
13750            &mut rx,
13751            hello_frame("m", PROTOCOL_VERSION, 1),
13752        )
13753        .await;
13754        let ack = parse_ack(&ack);
13755        assert!(
13756            ack.subc_capabilities
13757                .iter()
13758                .any(|c| c == "route-role-versions/v1"),
13759            "{:?}",
13760            ack.subc_capabilities
13761        );
13762
13763        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13764        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13765        let reply = handler
13766            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13767            .await
13768            .unwrap()
13769            .pop()
13770            .unwrap();
13771        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13772            serde_json::from_slice(&reply.body).unwrap()
13773        else {
13774            panic!("not a server.describe reply");
13775        };
13776        assert!(
13777            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13778            "{capabilities:?}"
13779        );
13780    }
13781
13782    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
13783    /// catalog entry without `events` (the module declares none) from a daemon
13784    /// that never reports declared events.
13785    #[tokio::test]
13786    async fn catalog_events_capability_is_advertised() {
13787        let handler = ControlHandler::new(Arc::new(Registry::default()));
13788        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13789        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13790        let reply = handler
13791            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13792            .await
13793            .unwrap()
13794            .pop()
13795            .unwrap();
13796        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13797            serde_json::from_slice(&reply.body).unwrap()
13798        else {
13799            panic!("not a server.describe reply");
13800        };
13801        assert!(
13802            capabilities
13803                .iter()
13804                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13805            "{capabilities:?}"
13806        );
13807    }
13808
13809    #[tokio::test]
13810    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13811        let handler = ControlHandler::new(Arc::new(Registry::default()));
13812        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13813        let ack = hello_via_sink(
13814            &handler,
13815            &module,
13816            &mut module_rx,
13817            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13818        )
13819        .await;
13820        let ack = parse_ack(&ack);
13821        assert!(
13822            ack.subc_capabilities
13823                .iter()
13824                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13825            "{:?}",
13826            ack.subc_capabilities
13827        );
13828        assert!(ack
13829            .subc_capabilities
13830            .iter()
13831            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13832
13833        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13834        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13835        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13836        let reply = handler
13837            .handle_control_frame(&client, frame)
13838            .await
13839            .unwrap()
13840            .pop()
13841            .unwrap();
13842        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13843            serde_json::from_slice(&reply.body).unwrap()
13844        else {
13845            panic!("not a server.describe reply");
13846        };
13847        assert!(
13848            capabilities
13849                .iter()
13850                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13851            "{capabilities:?}"
13852        );
13853        assert!(capabilities
13854            .iter()
13855            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13856    }
13857
13858    #[tokio::test]
13859    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13860        let handler = ControlHandler::new(Arc::new(Registry::default()));
13861        let (module, mut rx) = route_ctx(ConnectionId::new(156));
13862        let ack = hello_via_sink(
13863            &handler,
13864            &module,
13865            &mut rx,
13866            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13867        )
13868        .await;
13869        assert!(parse_ack(&ack)
13870            .subc_capabilities
13871            .iter()
13872            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13873        assert!(parse_ack(&ack)
13874            .subc_capabilities
13875            .iter()
13876            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13877        let (client, _rx) = route_ctx(ConnectionId::new(157));
13878        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13879        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13880        let reply = handler
13881            .handle_control_frame(&client, frame)
13882            .await
13883            .unwrap()
13884            .pop()
13885            .unwrap();
13886        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13887            serde_json::from_slice(&reply.body).unwrap()
13888        else {
13889            panic!("server.describe reply")
13890        };
13891        assert!(capabilities
13892            .iter()
13893            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13894        assert!(capabilities
13895            .iter()
13896            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13897    }
13898
13899    #[tokio::test]
13900    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13901        let handler = ControlHandler::new(Arc::new(Registry::default()));
13902        let (module, mut rx) = route_ctx(ConnectionId::new(158));
13903        let ack = hello_via_sink(
13904            &handler,
13905            &module,
13906            &mut rx,
13907            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13908        )
13909        .await;
13910        assert!(parse_ack(&ack)
13911            .subc_capabilities
13912            .iter()
13913            .any(|cap| cap == "scope-workdir/v1"));
13914        let (client, _rx) = route_ctx(ConnectionId::new(159));
13915        let frame = Frame::build(
13916            FrameType::Request,
13917            control_flags(),
13918            0,
13919            0,
13920            5,
13921            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13922        )
13923        .unwrap();
13924        let reply = handler
13925            .handle_control_frame(&client, frame)
13926            .await
13927            .unwrap()
13928            .remove(0);
13929        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13930            serde_json::from_slice(&reply.body).unwrap()
13931        else {
13932            panic!("server.describe reply")
13933        };
13934        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13935    }
13936
13937    #[tokio::test]
13938    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13939        let handler = ControlHandler::new(Arc::new(Registry::default()));
13940        let (module, mut rx) = route_ctx(ConnectionId::new(160));
13941        let ack = hello_via_sink(
13942            &handler,
13943            &module,
13944            &mut rx,
13945            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13946        )
13947        .await;
13948        assert!(parse_ack(&ack)
13949            .subc_capabilities
13950            .iter()
13951            .any(|cap| cap == "worker-runs/v1"));
13952        let (client, _rx) = route_ctx(ConnectionId::new(161));
13953        let frame = Frame::build(
13954            FrameType::Request,
13955            control_flags(),
13956            0,
13957            0,
13958            5,
13959            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13960        )
13961        .unwrap();
13962        let reply = handler
13963            .handle_control_frame(&client, frame)
13964            .await
13965            .unwrap()
13966            .remove(0);
13967        let ClientControlResponse::ServerDescribe { capabilities, .. } =
13968            serde_json::from_slice(&reply.body).unwrap()
13969        else {
13970            panic!("server.describe reply")
13971        };
13972        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13973    }
13974
13975    #[tokio::test]
13976    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13977        let handler = ControlHandler::new(Arc::new(Registry::default()));
13978        let (module, mut rx) = route_ctx(ConnectionId::new(154));
13979        let ack = hello_via_sink(
13980            &handler,
13981            &module,
13982            &mut rx,
13983            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13984        )
13985        .await;
13986        assert!(parse_ack(&ack)
13987            .subc_capabilities
13988            .iter()
13989            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13990        let (client, _rx) = route_ctx(ConnectionId::new(155));
13991        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13992        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13993        let reply = handler
13994            .handle_control_frame(&client, frame)
13995            .await
13996            .unwrap()
13997            .pop()
13998            .unwrap();
13999        let ClientControlResponse::ServerDescribe { capabilities, .. } =
14000            serde_json::from_slice(&reply.body).unwrap()
14001        else {
14002            panic!("not a server.describe reply")
14003        };
14004        assert!(capabilities
14005            .iter()
14006            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
14007    }
14008
14009    #[tokio::test]
14010    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
14011        let registry = Arc::new(Registry::default());
14012        let forwarding = Arc::new(ForwardingTable::default());
14013        let handler =
14014            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14015                .with_health_probe_timeout(Duration::from_secs(5));
14016        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
14017        hello_via_sink(
14018            &handler,
14019            &module_ctx,
14020            &mut module_rx,
14021            non_routable_hello_frame_with_control_ops(
14022                "mcp",
14023                300,
14024                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14025            ),
14026        )
14027        .await;
14028        assert!(registry
14029            .get_module("mcp")
14030            .unwrap()
14031            .unwrap()
14032            .manifest
14033            .provides
14034            .is_empty());
14035
14036        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
14037        let route_response = handler
14038            .handle_control_frame(
14039                &route_client_ctx,
14040                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
14041            )
14042            .await
14043            .unwrap();
14044        assert_eq!(route_response[0].header.ty, FrameType::Error);
14045        assert_eq!(
14046            parse_error(&route_response[0])["code"],
14047            "target_unavailable"
14048        );
14049        assert!(parse_error(&route_response[0])["message"]
14050            .as_str()
14051            .unwrap()
14052            .contains("does not provide the requested target"));
14053        assert!(module_rx.try_recv().is_err());
14054
14055        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
14056        let health_handler = handler.clone();
14057        let health_task = tokio::spawn(async move {
14058            health_handler
14059                .handle_control_frame(
14060                    &health_client_ctx,
14061                    supervisor_health_probe_frame(302, "mcp"),
14062                )
14063                .await
14064                .unwrap()
14065        });
14066        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14067            .await
14068            .unwrap()
14069            .unwrap();
14070        assert_eq!(
14071            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
14072            ModuleControlRequest::HealthCheck {}
14073        );
14074        handler
14075            .handle_control_frame(
14076                &module_ctx,
14077                health_response(health_frame.header.corr, HealthStatus::Ok),
14078            )
14079            .await
14080            .unwrap();
14081        let health_response = health_task.await.unwrap();
14082        assert_eq!(health_response[0].header.ty, FrameType::Response);
14083        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
14084            ClientControlResponse::SupervisorHealthProbe {
14085                module_id, status, ..
14086            } => {
14087                assert_eq!(module_id, "mcp");
14088                assert_eq!(status, HealthStatus::Ok);
14089            }
14090            other => panic!("unexpected health response: {other:?}"),
14091        }
14092
14093        // Exercise the forwarding cleanup path directly while leaving the registry
14094        // advertisement in place. If cleanup leaves a stale control sink behind,
14095        // the next probe will enqueue onto it and wait for the long probe timeout
14096        // instead of returning an immediate no-connection error.
14097        forwarding
14098            .cleanup_connection(module_ctx.connection_id)
14099            .unwrap();
14100        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
14101        let cleanup_response = tokio::time::timeout(
14102            Duration::from_millis(200),
14103            handler.handle_control_frame(
14104                &cleanup_probe_ctx,
14105                supervisor_health_probe_frame(303, "mcp"),
14106            ),
14107        )
14108        .await
14109        .expect("probe should fail immediately when the control lane is gone")
14110        .unwrap();
14111        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
14112        assert_eq!(
14113            parse_error(&cleanup_response[0])["code"],
14114            "target_unavailable"
14115        );
14116        assert!(parse_error(&cleanup_response[0])["message"]
14117            .as_str()
14118            .unwrap()
14119            .contains("no module connection"));
14120
14121        handler
14122            .cleanup_connection(module_ctx.connection_id)
14123            .unwrap();
14124    }
14125
14126    #[tokio::test]
14127    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
14128        let registry = Arc::new(Registry::default());
14129        let supervisor_handle = SupervisorHandle::new();
14130        let supervisor =
14131            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
14132                .with_handle(supervisor_handle.clone())
14133                .with_connection_file_path(
14134                    std::env::temp_dir()
14135                        .join(format!("subc-route-open-warming-{}", std::process::id())),
14136                );
14137        let module = supervisor
14138            .supervise_configured(
14139                ModuleSpec {
14140                    carry_containment: false,
14141                    module_id: "warming".to_string(),
14142                    program: fake_aft_stub_path(),
14143                    args: Vec::new(),
14144                    env: Vec::new(),
14145                    reserved: false,
14146                    reserved_prefixes: Vec::new(),
14147                    protocol: ModuleProtocol::Subc,
14148                    overlap: Default::default(),
14149                },
14150                true,
14151            )
14152            .unwrap();
14153        assert_eq!(module.state().unwrap(), ModuleState::Running);
14154
14155        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14156        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
14157        let response = handler
14158            .handle_control_frame(
14159                &ctx,
14160                route_open_frame(304, "warming", unique_project_root("warming")),
14161            )
14162            .await
14163            .unwrap();
14164        module.stop().await.unwrap();
14165
14166        assert_eq!(response[0].header.ty, FrameType::Error);
14167        let error = parse_error(&response[0]);
14168        assert_eq!(error["code"], "module_warming");
14169        assert!(error["message"]
14170            .as_str()
14171            .unwrap()
14172            .contains("state=running, enabled=true, live=false"));
14173    }
14174
14175    #[test]
14176    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
14177        let handler = ControlHandler::new(Arc::new(Registry::default()));
14178        let capture = EventCapture::default();
14179        let _subscriber =
14180            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14181        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
14182        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
14183        let pending = (0..limit).collect::<Vec<_>>();
14184        let response = handler
14185            .route_open_capacity_refusal(
14186                &ctx,
14187                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14188                "busy",
14189                pending.len(),
14190                limit,
14191            )
14192            .unwrap();
14193        assert_eq!(parse_error(&response)["code"], "target_unavailable");
14194        let event = capture
14195            .events()
14196            .into_iter()
14197            .find(|event| {
14198                event.target == "control"
14199                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14200            })
14201            .expect("connection admission refusal event");
14202        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14203        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14204        // Refused before `handle_route_open` parses the request, so the caller
14205        // fields are read from the frame itself.
14206        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14207        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14208    }
14209
14210    #[test]
14211    fn route_open_target_cap_logs_admission_reason_and_capacity() {
14212        let handler = ControlHandler::new(Arc::new(Registry::default()));
14213        let capture = EventCapture::default();
14214        let _subscriber =
14215            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14216        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14217        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14218        let guards = (0..limit)
14219            .map(|_| {
14220                handler
14221                    .route_bind_concurrency
14222                    .try_admit("busy", limit)
14223                    .unwrap()
14224            })
14225            .collect::<Vec<_>>();
14226        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14227            Err(in_flight) => in_flight,
14228            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14229        };
14230        let response = handler
14231            .route_open_target_capacity_refusal(
14232                &ctx,
14233                &RouteOpenCaller::unknown(),
14234                &route_open_frame(397, "busy", unique_project_root("target-cap")),
14235                "busy",
14236                in_flight,
14237            )
14238            .unwrap();
14239        assert_eq!(parse_error(&response)["code"], "target_unavailable");
14240        let event = capture
14241            .events()
14242            .into_iter()
14243            .find(|event| {
14244                event.target == "control"
14245                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14246            })
14247            .expect("target admission refusal event");
14248        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14249        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14250        drop(guards);
14251    }
14252
14253    #[cfg(unix)]
14254    #[tokio::test(start_paused = true)]
14255    async fn supervisor_list_shows_a_failed_modules_next_retry() {
14256        let registry = Arc::new(Registry::default());
14257        let handle = SupervisorHandle::new();
14258        let supervisor =
14259            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14260                .with_handle(handle.clone());
14261        let module = supervisor
14262            .spawn(crate::ModuleSpec {
14263                carry_containment: false,
14264                module_id: "retry-list".into(),
14265                program: PathBuf::from("/bin/sh"),
14266                args: vec!["-c".into(), "exit 17".into()],
14267                env: vec![],
14268                reserved: false,
14269                reserved_prefixes: vec![],
14270                protocol: ModuleProtocol::None,
14271                overlap: Default::default(),
14272            })
14273            .unwrap();
14274        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14275        while module.state().unwrap() != crate::ModuleState::Failed {
14276            assert!(
14277                std::time::Instant::now() < deadline,
14278                "waiting for budget failure"
14279            );
14280            tokio::task::yield_now().await;
14281        }
14282        let handler = ControlHandler::new(registry).with_supervisor(handle);
14283        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14284        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14285        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14286        assert_eq!(body["modules"][0]["state"], "failed");
14287        let retry = body["modules"][0]["next_retry_at_ms"]
14288            .as_u64()
14289            .expect("scheduled retry on the wire");
14290        let now = std::time::SystemTime::now()
14291            .duration_since(std::time::UNIX_EPOCH)
14292            .unwrap()
14293            .as_millis() as u64;
14294        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14295        module.set_enabled(false).await.unwrap();
14296        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14297        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14298        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14299        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14300    }
14301
14302    #[tokio::test]
14303    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14304        let _guard = crate::subtree_resources::fault_read_guard().await;
14305        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14306        let supervisor =
14307            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14308        let module = supervisor
14309            .spawn(crate::ModuleSpec {
14310                carry_containment: false,
14311                module_id: "pid-read-budget".into(),
14312                program: std::env::current_exe().unwrap(),
14313                args: vec![
14314                    "--exact".into(),
14315                    "control::tests::pid_resource_read_fixture".into(),
14316                ],
14317                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14318                    .into_iter()
14319                    .map(|key| (key.into(), dir.display().to_string()))
14320                    .chain(std::iter::once((
14321                        "SUBC_SUBTREE_PID_FIXTURE".into(),
14322                        "1".into(),
14323                    )))
14324                    .collect(),
14325                reserved: false,
14326                reserved_prefixes: Vec::new(),
14327                protocol: ModuleProtocol::None,
14328                overlap: Default::default(),
14329            })
14330            .unwrap();
14331        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14332        while !matches!(
14333            module.child_resource_usage(),
14334            subc_control::ChildResourceUsage::Measured(_)
14335        ) {
14336            assert!(
14337                tokio::time::Instant::now() < ready_deadline,
14338                "fixture process was not measurable"
14339            );
14340            tokio::time::sleep(Duration::from_millis(10)).await;
14341        }
14342        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14343        let reading = crate::subtree_resources::read_process(module.clone()).await;
14344        module.stop().await.unwrap();
14345        assert!(
14346            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14347            "{reading:?}"
14348        );
14349    }
14350
14351    #[test]
14352    fn pid_resource_read_fixture() {
14353        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14354            loop {
14355                std::thread::park_timeout(Duration::from_secs(600));
14356            }
14357        }
14358    }
14359
14360    #[tokio::test]
14361    async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14362        let registry = Arc::new(Registry::default());
14363        let handle = SupervisorHandle::new();
14364        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14365            .with_handle(handle.clone());
14366        supervisor
14367            .supervise_configured(
14368                crate::ModuleSpec {
14369                    module_id: "disabled-resource-reading".into(),
14370                    program: std::env::current_exe().unwrap(),
14371                    args: Vec::new(),
14372                    env: Vec::new(),
14373                    reserved: false,
14374                    reserved_prefixes: Vec::new(),
14375                    protocol: ModuleProtocol::None,
14376                    overlap: Default::default(),
14377                    carry_containment: false,
14378                },
14379                false,
14380            )
14381            .unwrap();
14382        let handler = ControlHandler::new(registry).with_supervisor(handle);
14383        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14384        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14385        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14386        assert!(body["modules"][0].get("tree").is_none());
14387        assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14388    }
14389
14390    #[tokio::test]
14391    async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14392        let _guard = crate::subtree_resources::fault_read_guard().await;
14393        let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14394        let registry = Arc::new(Registry::default());
14395        let handle = SupervisorHandle::new();
14396        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14397            .with_handle(handle.clone());
14398        let module = supervisor
14399            .spawn(crate::ModuleSpec {
14400                carry_containment: false,
14401                module_id: "list-process-priority".into(),
14402                program: std::env::current_exe().unwrap(),
14403                args: vec![
14404                    "--exact".into(),
14405                    "control::tests::pid_resource_read_fixture".into(),
14406                ],
14407                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14408                    .into_iter()
14409                    .map(|key| (key.into(), dir.display().to_string()))
14410                    .chain(std::iter::once((
14411                        "SUBC_SUBTREE_PID_FIXTURE".into(),
14412                        "1".into(),
14413                    )))
14414                    .collect(),
14415                reserved: false,
14416                reserved_prefixes: Vec::new(),
14417                protocol: ModuleProtocol::None,
14418                overlap: Default::default(),
14419            })
14420            .unwrap();
14421        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14422        loop {
14423            let image = module.running_image_agreement().await;
14424            if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14425                break;
14426            }
14427            assert!(
14428                tokio::time::Instant::now() < ready_deadline,
14429                "the fixture process never became observable: {image:?}"
14430            );
14431            tokio::time::sleep(Duration::from_millis(10)).await;
14432        }
14433        let (target, release, _) = crate::subtree_resources::held();
14434        module.set_subtree_target_for_test(target);
14435        let handler = ControlHandler::new(registry).with_supervisor(handle);
14436        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14437        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14438        drop(release);
14439        module.stop().await.unwrap();
14440        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14441        assert_eq!(
14442            body["modules"][0]["resources"]["status"], "measured",
14443            "the optional tree timeout must not discard the process reading: {body}"
14444        );
14445        assert!(body["modules"][0].get("tree").is_none());
14446        assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14447        assert_eq!(
14448            body["modules"][0]["pending_reload"]["image"]["status"], "match",
14449            "the tree timeout must not hide a completed image observation"
14450        );
14451    }
14452
14453    #[cfg(any(target_os = "linux", target_os = "macos"))]
14454    #[tokio::test]
14455    async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14456        let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14457            .with_daemon_provenance(
14458                std::process::id(),
14459                0,
14460                Some(std::env::current_exe().unwrap()),
14461                None,
14462                None,
14463            );
14464        let (probe, release, began) = ExecutableIdentityProbe::held();
14465        handler.daemon_provenance.probe = probe;
14466        let observe = |handler: ControlHandler, corr| {
14467            tokio::spawn(async move {
14468                let (ctx, _) = route_ctx(ConnectionId::new(corr));
14469                let frame =
14470                    Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14471                        .unwrap();
14472                handler
14473                    .handle_supervisor_provenance(&ctx, frame, None)
14474                    .await
14475                    .unwrap()
14476            })
14477        };
14478        let explicit = observe(handler.clone(), 114);
14479        began.await.unwrap();
14480        let concurrent = observe(handler.clone(), 115);
14481        // Keep explicit provenance waiting on its first image observation beyond
14482        // the list deadline; release it only after the concurrent list replies.
14483        tokio::time::sleep(Duration::from_millis(300)).await;
14484        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14485        let list = tokio::time::timeout(
14486            Duration::from_secs(1),
14487            handler.handle_supervisor_list(frame),
14488        )
14489        .await;
14490        let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14491        drop(release);
14492        let replies = explicit.await.unwrap();
14493        let concurrent_replies = concurrent.await.unwrap();
14494        assert!(
14495            explicit_was_pending,
14496            "explicit provenance must await the full observation"
14497        );
14498        let list = list
14499            .expect("a concurrent explicit probe must not stall list")
14500            .unwrap();
14501        let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14502        assert_eq!(list_body["op"], "supervisor.list");
14503        for replies in [replies, concurrent_replies] {
14504            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14505            assert_eq!(
14506                body["daemon"]["daemon_observed"]["running_image"]["status"],
14507                "match"
14508            );
14509        }
14510    }
14511
14512    #[tokio::test]
14513    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14514        let _guard = crate::subtree_resources::fault_read_guard().await;
14515        let registry = Arc::new(Registry::default());
14516        let handle = SupervisorHandle::new();
14517        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14518            .with_handle(handle.clone());
14519        let module = supervisor
14520            .supervise_configured(
14521                crate::ModuleSpec {
14522                    carry_containment: false,
14523                    module_id: "memory-read".into(),
14524                    program: PathBuf::from("disabled-module"),
14525                    args: Vec::new(),
14526                    env: Vec::new(),
14527                    reserved: false,
14528                    reserved_prefixes: Vec::new(),
14529                    protocol: ModuleProtocol::Subc,
14530                    overlap: Default::default(),
14531                },
14532                true,
14533            )
14534            .unwrap();
14535        let (target, release, began) = crate::subtree_resources::held();
14536        // Optional list work may be skipped when its deadline is already spent.
14537        // Establish the stalled fixture before asking the list to observe it.
14538        let stalled_target = target.clone();
14539        let stalled = tokio::task::spawn_blocking(move || {
14540            stalled_target.tree(&subc_os::ProcessTable::default())
14541        });
14542        began.await.unwrap();
14543        module.set_subtree_target_for_test(target);
14544        let handler = ControlHandler::new(registry).with_supervisor(handle);
14545        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14546        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14547        let replies = handler
14548            .handle_client_control_request(
14549                &ctx,
14550                frame,
14551                ClientControlRequest::SupervisorList {},
14552                &mut None,
14553            )
14554            .await
14555            .unwrap();
14556        assert!(
14557            replies.is_empty(),
14558            "list reply must be detached from the connection reader"
14559        );
14560        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14561        let replies = handler
14562            .handle_client_control_request(
14563                &ctx,
14564                frame,
14565                ClientControlRequest::SupervisorSpawnSnapshot {},
14566                &mut None,
14567            )
14568            .await
14569            .unwrap();
14570        assert_eq!(
14571            replies.len(),
14572            1,
14573            "another request must complete while the subtree read stalls"
14574        );
14575        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14576            .await
14577            .unwrap()
14578            .unwrap();
14579        release.send(()).unwrap();
14580        drop(release);
14581        stalled.await.unwrap();
14582        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14583        assert!(
14584            body["modules"][0].get("tree").is_none(),
14585            "an unknown tree reading must be omitted after the bounded read expires"
14586        );
14587        // Lists export completed work rather than measuring synchronously.
14588        // Complete a refresh after the fixture is released before asserting
14589        // that a later list contains its measured tree.
14590        handler
14591            .resource_tracker
14592            .read(handler.supervisor.list(), false, "released_tree_fixture")
14593            .await;
14594        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14595        let replies = handler.handle_supervisor_list(frame).await.unwrap();
14596        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14597        assert_eq!(
14598            body["modules"][0]["tree"]["source"], "process_tree",
14599            "the next request must export the completed refresh"
14600        );
14601        assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14602    }
14603
14604    #[tokio::test]
14605    async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
14606        use subc_control::{
14607            AgentCommandResources, ChildMemoryKind, ChildResourceReading,
14608            ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
14609            TreeReading, TreeSource,
14610        };
14611        let registry = Arc::new(Registry::default());
14612        let handle = SupervisorHandle::new();
14613        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14614            .with_handle(handle.clone());
14615        supervisor
14616            .supervise_configured(
14617                crate::ModuleSpec {
14618                    carry_containment: false,
14619                    module_id: "budget-worker".into(),
14620                    program: PathBuf::from("disabled-module"),
14621                    args: Vec::new(),
14622                    env: Vec::new(),
14623                    reserved: false,
14624                    reserved_prefixes: Vec::new(),
14625                    protocol: ModuleProtocol::Subc,
14626                    overlap: Default::default(),
14627                },
14628                true,
14629            )
14630            .unwrap();
14631        let handler = ControlHandler::new(registry).with_supervisor(handle);
14632        let now = u64::try_from(
14633            std::time::SystemTime::now()
14634                .duration_since(std::time::UNIX_EPOCH)
14635                .unwrap()
14636                .as_millis(),
14637        )
14638        .unwrap();
14639        let measured = now - 1_000;
14640        let hour = CpuLastHour::new(100, 60_000);
14641        let daemon = DaemonResources::new(7, 1, "test")
14642            .with_measured_at_ms(Some(measured))
14643            .with_resources(Some(ChildResourceUsage::Measured(
14644                ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
14645            )))
14646            .with_cpu_last_hour(Some(hour.clone()));
14647        let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
14648            .with_measured_at_ms(Some(measured))
14649            .with_cpu_last_hour(Some(hour.clone()));
14650        let tree = TreeReading::new(
14651            150,
14652            50,
14653            ChildMemoryKind::ResidentSet,
14654            1,
14655            TreeSource::ProcessTree,
14656        )
14657        .with_measured_at_ms(Some(measured));
14658        handler.resource_tracker.seed_committed_for_test(
14659            &handler.supervisor.list(),
14660            crate::resource_totals::Readings {
14661                modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
14662                daemon: Some(daemon),
14663                agent_commands: Some(vec![agents]),
14664                ..Default::default()
14665            },
14666            now,
14667        );
14668        list_diagnostics::scope(async {
14669            tokio::time::sleep(Duration::from_millis(110)).await;
14670            assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
14671            let request =
14672                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14673            let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
14674            let ClientControlResponse::SupervisorList {
14675                modules,
14676                daemon,
14677                agent_commands,
14678                ..
14679            } = serde_json::from_slice(&replies[0].body).unwrap()
14680            else {
14681                panic!("expected a list response")
14682            };
14683            assert!(matches!(
14684                modules[0].resources,
14685                Some(ChildResourceUsage::Unavailable {
14686                    reason: ChildResourceUnavailableReason::Unreadable
14687                })
14688            ));
14689            assert_eq!(
14690                (
14691                    modules[0].tree.as_ref().unwrap().cpu_ms,
14692                    modules[0].tree.as_ref().unwrap().measured_at_ms
14693                ),
14694                (150, Some(measured))
14695            );
14696            let daemon = daemon.unwrap();
14697            let agents = agent_commands.unwrap();
14698            assert_eq!(daemon.measured_at_ms, Some(measured));
14699            assert_eq!(
14700                (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
14701                (1, 50, Some(measured))
14702            );
14703            for hour in [
14704                modules[0].cpu_last_hour.as_ref(),
14705                daemon.cpu_last_hour.as_ref(),
14706                agents[0].cpu_last_hour.as_ref(),
14707            ] {
14708                let hour = hour.unwrap();
14709                assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
14710            }
14711        })
14712        .await;
14713    }
14714
14715    #[tokio::test]
14716    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14717        let registry = Arc::new(Registry::default());
14718        let handle = SupervisorHandle::new();
14719        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14720            .with_handle(handle.clone());
14721        supervisor
14722            .supervise_configured(
14723                crate::ModuleSpec {
14724                    carry_containment: false,
14725                    module_id: "drift".into(),
14726                    program: PathBuf::from("disabled-module"),
14727                    args: Vec::new(),
14728                    env: Vec::new(),
14729                    reserved: false,
14730                    reserved_prefixes: Vec::new(),
14731                    protocol: ModuleProtocol::Subc,
14732                    overlap: Default::default(),
14733                },
14734                false,
14735            )
14736            .unwrap();
14737        let handler = ControlHandler::new(registry).with_supervisor(handle);
14738        for configured in [None, Some(false), Some(true)] {
14739            handler.capability_evaluator.configure(
14740                configured.map(|enabled| ("drift".to_string(), enabled)),
14741                BTreeMap::new(),
14742            );
14743            let frame =
14744                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14745            let replies = handler
14746                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14747                .await
14748                .unwrap();
14749            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14750            assert!(body.get("configured_enabled").is_none());
14751            assert_eq!(body["op"], "supervisor.ack");
14752            let frame =
14753                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14754            let replies = handler.handle_supervisor_list(frame).await.unwrap();
14755            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14756            assert_eq!(
14757                body["modules"][0].get("configured_enabled"),
14758                configured.map(Value::Bool).as_ref()
14759            );
14760            assert!(body["modules"][0].get("tree").is_none());
14761            assert!(body["modules"][0].get("subtree").is_none());
14762        }
14763    }
14764
14765    #[tokio::test(flavor = "current_thread")]
14766    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14767        let handler = ControlHandler::new(Arc::new(Registry::default()));
14768        let capture = EventCapture::default();
14769        let _subscriber =
14770            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14771        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14772        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14773
14774        let _ = handler
14775            .handle_client_control_request(
14776                &ctx,
14777                frame,
14778                ClientControlRequest::SupervisorSetEnabled {
14779                    module_id: "broca".to_string(),
14780                    enabled: false,
14781                },
14782                &mut None,
14783            )
14784            .await
14785            .unwrap();
14786
14787        let events = capture
14788            .events()
14789            .into_iter()
14790            .filter(|event| {
14791                event.target == "control"
14792                    && event.fields.get("message").map(String::as_str)
14793                        == Some("supervisor request received")
14794            })
14795            .collect::<Vec<_>>();
14796        assert_eq!(events.len(), 1, "one supervisor request log line");
14797        let event = &events[0];
14798        assert_eq!(event.level, tracing::Level::INFO);
14799        assert_eq!(
14800            event.fields.get("op"),
14801            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14802        );
14803        assert_eq!(
14804            event.fields.get("module_id"),
14805            Some(&"\"broca\"".to_string())
14806        );
14807        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14808        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14809        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14810    }
14811
14812    #[tokio::test(flavor = "current_thread")]
14813    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14814        let registry = Arc::new(Registry::default());
14815        let module_connection = ConnectionId::new(303);
14816        registry
14817            .register_with_control_ops(
14818                manifest("aft", PROTOCOL_VERSION),
14819                PROTOCOL_VERSION,
14820                module_connection,
14821                module_baseline_control_ops(),
14822            )
14823            .unwrap();
14824        let handler = ControlHandler::new(registry);
14825        let capture = EventCapture::default();
14826        let _subscriber =
14827            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14828        let (ctx, _rx) = route_ctx(module_connection);
14829        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14830
14831        let _ = handler
14832            .handle_client_control_request(
14833                &ctx,
14834                frame,
14835                ClientControlRequest::SupervisorSetEnabled {
14836                    module_id: "broca".to_string(),
14837                    enabled: false,
14838                },
14839                &mut None,
14840            )
14841            .await
14842            .unwrap();
14843
14844        let callers = capture
14845            .events()
14846            .into_iter()
14847            .filter(|event| {
14848                event.fields.get("message").map(String::as_str)
14849                    == Some("supervisor request received")
14850            })
14851            .map(|event| event.fields.get("caller").cloned())
14852            .collect::<Vec<_>>();
14853        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14854    }
14855
14856    #[tokio::test(flavor = "current_thread")]
14857    async fn supervisor_list_does_not_log_request_received() {
14858        let handler = ControlHandler::new(Arc::new(Registry::default()));
14859        let capture = EventCapture::default();
14860        let _subscriber =
14861            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14862        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14863        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14864
14865        handler
14866            .handle_client_control_request(
14867                &ctx,
14868                frame,
14869                ClientControlRequest::SupervisorList {},
14870                &mut None,
14871            )
14872            .await
14873            .unwrap();
14874
14875        assert!(capture.events().into_iter().all(|event| {
14876            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14877        }));
14878    }
14879
14880    #[tokio::test(flavor = "current_thread")]
14881    async fn supervisor_rescan_preview_does_not_log_request_received() {
14882        let handler = ControlHandler::new(Arc::new(Registry::default()));
14883        let capture = EventCapture::default();
14884        let _subscriber =
14885            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14886        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14887        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14888
14889        handler
14890            .handle_client_control_request(
14891                &ctx,
14892                frame,
14893                ClientControlRequest::SupervisorRescan { preview: true },
14894                &mut None,
14895            )
14896            .await
14897            .unwrap();
14898
14899        assert!(capture.events().into_iter().all(|event| {
14900            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14901        }));
14902    }
14903
14904    struct ApproveOperator;
14905    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14906        fn prompt(
14907            &self,
14908            _: &str,
14909            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14910        ) -> crate::operator_confirm::ProviderResult {
14911            crate::operator_confirm::ProviderResult::Approved
14912        }
14913    }
14914
14915    #[tokio::test]
14916    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14917        assert_eq!(
14918            ControlHandler::default().privileged_operator_prompt,
14919            PrivilegedOperatorPrompt::Log
14920        );
14921        for (initial, changed, running) in [
14922            ("log", "enforce", PrivilegedOperatorPrompt::Log),
14923            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14924        ] {
14925            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14926            let path = dir.join("subc.jsonc");
14927            std::fs::write(
14928                &path,
14929                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14930            )
14931            .unwrap();
14932            let config = crate::daemon_config::load(&path).unwrap().unwrap();
14933            let registry = Arc::new(Registry::default());
14934            let handle = SupervisorHandle::new();
14935            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14936                .with_handle(handle.clone());
14937            let handler = ControlHandler::new(registry)
14938                .with_supervisor(handle)
14939                .with_privileged_operator_prompt(config.privileged_operator_prompt)
14940                .with_supervisor_rescan(supervisor, &path, config.port);
14941            handler.forwarding.operator_confirms().configure(
14942                Arc::new(ApproveOperator),
14943                crate::operator_confirm::OperatorLimits::default(),
14944            );
14945            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14946
14947            for (disk_mode, needs_restart) in [
14948                (Some(initial), false),
14949                (Some(changed), true),
14950                (Some(changed), true),
14951                (None, running == PrivilegedOperatorPrompt::Enforce),
14952                (Some(initial), false),
14953            ] {
14954                let mut doc = json!({"version":1});
14955                if let Some(mode) = disk_mode {
14956                    doc["privileged_operator_prompt"] = json!(mode);
14957                }
14958                std::fs::write(&path, doc.to_string()).unwrap();
14959                for preview in [true, false] {
14960                    let request = ClientControlRequest::SupervisorRescan { preview };
14961                    let frame = Frame::build(
14962                        FrameType::Request,
14963                        control_flags(),
14964                        0,
14965                        0,
14966                        1,
14967                        serde_json::to_vec(&request).unwrap(),
14968                    )
14969                    .unwrap();
14970                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14971                    assert_eq!(frames.len(), 1);
14972                    let ClientControlResponse::SupervisorRescan { result } =
14973                        serde_json::from_slice(&frames[0].body).unwrap()
14974                    else {
14975                        panic!("expected a rescan response");
14976                    };
14977                    let expected = if needs_restart {
14978                        vec!["privileged_operator_prompt"]
14979                    } else {
14980                        Vec::new()
14981                    };
14982                    assert_eq!(result.restart_required, expected);
14983                    assert_eq!(handler.privileged_operator_prompt, running);
14984                    assert_eq!(
14985                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14986                        running
14987                    );
14988                }
14989            }
14990        }
14991    }
14992
14993    #[tokio::test]
14994    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14995        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14996        let path = dir.join("subc.jsonc");
14997        let registry = Arc::new(Registry::default());
14998        let handle = SupervisorHandle::new();
14999        let supervisor =
15000            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
15001        let handler = ControlHandler::new(registry)
15002            .with_supervisor(handle)
15003            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
15004            .with_supervisor_rescan(supervisor, &path, None);
15005        handler.forwarding.operator_confirms().configure(
15006            Arc::new(ApproveOperator),
15007            crate::operator_confirm::OperatorLimits::default(),
15008        );
15009        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
15010        for preview in [true, false] {
15011            std::fs::write(
15012                &path,
15013                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
15014            )
15015            .unwrap();
15016            let request = ClientControlRequest::SupervisorRescan { preview };
15017            let frame = Frame::build(
15018                FrameType::Request,
15019                control_flags(),
15020                0,
15021                0,
15022                1,
15023                serde_json::to_vec(&request).unwrap(),
15024            )
15025            .unwrap();
15026            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
15027            assert_eq!(frames.len(), 1);
15028            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
15029            assert_eq!(
15030                handler.privileged_operator_prompt,
15031                PrivilegedOperatorPrompt::Enforce
15032            );
15033        }
15034    }
15035
15036    /// One wire code has several senders, so the refusal line names the check
15037    /// that refused. This drives the shared refusal path for ordinary refusals
15038    /// with an unregistered
15039    /// target and requires the branch label on the event.
15040    #[tokio::test]
15041    async fn route_open_refusal_names_the_check_that_refused() {
15042        let handler = ControlHandler::new(Arc::new(Registry::default()));
15043        let capture = EventCapture::default();
15044        let _subscriber =
15045            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15046        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15047        let response = handler
15048            .handle_control_frame(
15049                &ctx,
15050                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
15051            )
15052            .await
15053            .unwrap();
15054
15055        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15056        let event = capture
15057            .events()
15058            .into_iter()
15059            .find(|event| {
15060                event.target == "control"
15061                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15062            })
15063            .expect("route.open refusal event");
15064        assert_eq!(
15065            event.fields.get("reason"),
15066            Some(&"\"not_registered\"".to_string())
15067        );
15068    }
15069
15070    #[tokio::test]
15071    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
15072        let registry = Arc::new(Registry::default());
15073        let supervisor_handle = SupervisorHandle::new();
15074        let supervisor =
15075            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
15076                .with_handle(supervisor_handle.clone())
15077                .with_connection_file_path(std::env::temp_dir().join(format!(
15078                    "subc-route-open-refusal-info-{}",
15079                    std::process::id()
15080                )));
15081        let module = supervisor
15082            .supervise_configured(
15083                ModuleSpec {
15084                    carry_containment: false,
15085                    module_id: "warming".to_string(),
15086                    program: fake_aft_stub_path(),
15087                    args: Vec::new(),
15088                    env: Vec::new(),
15089                    reserved: false,
15090                    reserved_prefixes: Vec::new(),
15091                    protocol: ModuleProtocol::Subc,
15092                    overlap: Default::default(),
15093                },
15094                true,
15095            )
15096            .unwrap();
15097        assert_eq!(module.state().unwrap(), ModuleState::Running);
15098
15099        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15100        assert!(handler
15101            .counters()
15102            .snapshot()
15103            .get("route_open_refused_by_code")
15104            .is_none());
15105        let capture = EventCapture::default();
15106        let _subscriber =
15107            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15108        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
15109        let response = handler
15110            .handle_control_frame(
15111                &ctx,
15112                route_open_frame(394, "warming", unique_project_root("refusal-info")),
15113            )
15114            .await
15115            .unwrap();
15116        module.stop().await.unwrap();
15117
15118        assert_eq!(parse_error(&response[0])["code"], "module_warming");
15119        let event = capture
15120            .events()
15121            .into_iter()
15122            .find(|event| {
15123                event.target == "control"
15124                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
15125            })
15126            .expect("route.open refusal event");
15127        assert_eq!(
15128            event.fields.get("module_id"),
15129            Some(&"\"warming\"".to_string())
15130        );
15131        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
15132        assert_eq!(
15133            event.fields.get("reason"),
15134            Some(&"\"supervised_not_registered\"".to_string())
15135        );
15136        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
15137        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
15138        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
15139        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
15140        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
15141        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
15142        assert_eq!(
15143            handler.counters().snapshot()["route_open_refused_by_code"],
15144            json!({ "module_warming": 1 })
15145        );
15146    }
15147
15148    const OUTAGE_START: &str = "route.open refusing module: not serving";
15149    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
15150
15151    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
15152        capture
15153            .events()
15154            .into_iter()
15155            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
15156            .collect()
15157    }
15158
15159    fn supervise_stub(
15160        registry: &Arc<Registry>,
15161        module_id: &str,
15162        enabled: bool,
15163    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
15164        let supervisor_handle = SupervisorHandle::new();
15165        let supervisor =
15166            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
15167                .with_handle(supervisor_handle.clone())
15168                .with_connection_file_path(std::env::temp_dir().join(format!(
15169                    "subc-route-outage-{module_id}-{}",
15170                    std::process::id()
15171                )));
15172        let module = supervisor
15173            .supervise_configured(
15174                ModuleSpec {
15175                    carry_containment: false,
15176                    module_id: module_id.to_string(),
15177                    program: fake_aft_stub_path(),
15178                    args: Vec::new(),
15179                    env: Vec::new(),
15180                    reserved: false,
15181                    reserved_prefixes: Vec::new(),
15182                    protocol: ModuleProtocol::Subc,
15183                    overlap: Default::default(),
15184                },
15185                enabled,
15186            )
15187            .unwrap();
15188        (supervisor_handle, module)
15189    }
15190
15191    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
15192        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
15193            module_id: module_id.to_string(),
15194            drain_timeout_ms: Some(50),
15195        })
15196        .unwrap();
15197        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
15198    }
15199
15200    /// Two handlers built over one forwarding table must share one outage
15201    /// tracker; separate trackers would each log their own opening line for
15202    /// the same outage.
15203    #[test]
15204    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
15205        let registry = Arc::new(Registry::default());
15206        let forwarding = Arc::new(ForwardingTable::default());
15207        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15208        let second = ControlHandler::with_forwarding(registry, forwarding);
15209        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
15210    }
15211
15212    /// A client can name any module id it likes. Refusing an unknown one,
15213    /// however often, must not create outage state or outage lines, or the
15214    /// tracker would be a memory sink any client could fill.
15215    #[tokio::test(flavor = "current_thread")]
15216    async fn route_open_unknown_module_refusals_add_no_outage_state() {
15217        let handler = ControlHandler::new(Arc::new(Registry::default()));
15218        let capture = EventCapture::default();
15219        let _subscriber =
15220            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15221        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
15222        for corr in 0..8 {
15223            let response = handler
15224                .handle_control_frame(
15225                    &ctx,
15226                    route_open_frame(
15227                        380 + corr,
15228                        &format!("nobody-{corr}"),
15229                        unique_project_root("outage-unknown"),
15230                    ),
15231                )
15232                .await
15233                .unwrap();
15234            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15235        }
15236
15237        assert_eq!(handler.route_outages.tracked_module_count(), 0);
15238        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
15239        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
15240    }
15241
15242    /// Drives the refusal path end to end: a supervised module that served
15243    /// before and stopped being registered with no instruction to stop is a
15244    /// WARN, and the same module refused after an operator `supervisor.restart`
15245    /// is an INFO.
15246    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15247    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
15248        let registry = Arc::new(Registry::default());
15249        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
15250        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15251        let capture = EventCapture::default();
15252        let _subscriber =
15253            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15254        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
15255        // The stub never registers, so pretend it served once: otherwise every
15256        // refusal would fall in its startup window.
15257        handler.route_outages.record_accepted("outage-restart");
15258
15259        let response = handler
15260            .handle_control_frame(
15261                &ctx,
15262                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
15263            )
15264            .await
15265            .unwrap();
15266        assert_eq!(response[0].header.ty, FrameType::Error);
15267        let starts = outage_lines(&capture, OUTAGE_START);
15268        assert_eq!(starts.len(), 1, "{starts:?}");
15269        assert_eq!(starts[0].level, tracing::Level::WARN);
15270        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
15271        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
15272        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
15273        handler.route_outages.record_accepted("outage-restart");
15274        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
15275
15276        let restart = handler
15277            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
15278            .await
15279            .unwrap();
15280        assert_eq!(
15281            restart[0].header.ty,
15282            FrameType::Response,
15283            "{:?}",
15284            parse_error(&restart[0])
15285        );
15286        handler
15287            .handle_control_frame(
15288                &ctx,
15289                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
15290            )
15291            .await
15292            .unwrap();
15293        module.stop().await.unwrap();
15294
15295        let starts = outage_lines(&capture, OUTAGE_START);
15296        assert_eq!(starts.len(), 2, "{starts:?}");
15297        assert_eq!(starts[1].level, tracing::Level::INFO);
15298        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
15299    }
15300
15301    /// A restart refused before it touched the module (here: the module is
15302    /// disabled) must clear its operator mark, so the next real outage is
15303    /// still reported as a warning.
15304    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15305    async fn failed_operator_restart_leaves_no_operator_mark() {
15306        let registry = Arc::new(Registry::default());
15307        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
15308        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15309        let capture = EventCapture::default();
15310        let _subscriber =
15311            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15312        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15313        handler.route_outages.record_accepted("outage-disabled");
15314
15315        let restart = handler
15316            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15317            .await
15318            .unwrap();
15319        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15320        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15321
15322        handler
15323            .handle_control_frame(
15324                &ctx,
15325                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15326            )
15327            .await
15328            .unwrap();
15329        let starts = outage_lines(&capture, OUTAGE_START);
15330        assert_eq!(starts.len(), 1, "{starts:?}");
15331        assert_eq!(starts[0].level, tracing::Level::WARN);
15332    }
15333
15334    #[tokio::test(flavor = "current_thread")]
15335    async fn route_open_unknown_module_escapes_target_module_id() {
15336        let handler = ControlHandler::new(Arc::new(Registry::default()));
15337        let capture = EventCapture::default();
15338        let _subscriber =
15339            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15340        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15341        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15342        let response = handler
15343            .handle_control_frame(
15344                &ctx,
15345                route_open_frame(
15346                    395,
15347                    hostile_module_id,
15348                    unique_project_root("hostile-target-module-id"),
15349                ),
15350            )
15351            .await
15352            .unwrap();
15353
15354        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15355        let event = capture
15356            .events()
15357            .into_iter()
15358            .find(|event| {
15359                event.target == "control"
15360                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15361            })
15362            .expect("route.open unknown-module refusal event");
15363        let logged = event.fields.get("module_id").expect("module_id field");
15364        assert!(!logged.bytes().any(|byte| byte < 0x20));
15365        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15366    }
15367
15368    /// A route.open refusal is the daemon's own answer, so its ERROR carries
15369    /// the daemon-origin bit on the frame the client receives.
15370    #[tokio::test(flavor = "current_thread")]
15371    async fn route_open_refusal_carries_daemon_origin() {
15372        let handler = ControlHandler::new(Arc::new(Registry::default()));
15373        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15374        let response = handler
15375            .handle_control_frame(
15376                &ctx,
15377                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15378            )
15379            .await
15380            .unwrap();
15381
15382        assert_eq!(response.len(), 1);
15383        assert_eq!(response[0].header.ty, FrameType::Error);
15384        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15385        assert!(
15386            response[0].header.flags.is_daemon_origin(),
15387            "route.open refusal must carry daemon origin: {:?}",
15388            response[0].header.flags
15389        );
15390    }
15391
15392    /// The drain `module_reloading` refusal is daemon-synthesized for an
15393    /// endpoint that is reloading, so it carries the daemon-origin bit on the
15394    /// frame the client receives.
15395    #[tokio::test(flavor = "current_thread")]
15396    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15397        let registry = Arc::new(Registry::default());
15398        let forwarding = Arc::new(ForwardingTable::default());
15399        let handler =
15400            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15401        let module_connection = ConnectionId::new(95);
15402        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15403        hello_via_sink(
15404            &handler,
15405            &module_ctx,
15406            &mut module_rx,
15407            hello_frame("aft", PROTOCOL_VERSION, 395),
15408        )
15409        .await;
15410        forwarding
15411            .begin_module_drain("aft", RouteCloseReason::Restart)
15412            .unwrap()
15413            .expect("the module endpoint drains");
15414
15415        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15416        let response = handler
15417            .handle_control_frame(
15418                &client_ctx,
15419                route_open_frame(396, "aft", unique_project_root("drain-origin")),
15420            )
15421            .await
15422            .unwrap();
15423
15424        assert_eq!(response.len(), 1);
15425        assert_eq!(response[0].header.ty, FrameType::Error);
15426        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15427        assert!(
15428            response[0].header.flags.is_daemon_origin(),
15429            "drain module_reloading must carry daemon origin: {:?}",
15430            response[0].header.flags
15431        );
15432    }
15433
15434    /// A module that refuses a bind with a daemon code still cannot mint a
15435    /// daemon refusal: the relayed `module_reloading` body reaches the client
15436    /// with the daemon-origin bit clear.
15437    #[tokio::test(flavor = "current_thread")]
15438    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15439        let registry = Arc::new(Registry::default());
15440        let forwarding = Arc::new(ForwardingTable::default());
15441        let handler =
15442            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15443        let module_connection = ConnectionId::new(95);
15444        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15445        hello_via_sink(
15446            &handler,
15447            &module_ctx,
15448            &mut module_rx,
15449            hello_frame("aft", PROTOCOL_VERSION, 395),
15450        )
15451        .await;
15452
15453        let client_connection = ConnectionId::new(96);
15454        let (client_ctx, _client_rx) = route_ctx(client_connection);
15455        let (route_task, bind) = relay_route_open(
15456            &handler,
15457            client_connection,
15458            &client_ctx.egress,
15459            &mut module_rx,
15460            396,
15461            "aft",
15462            "spoofed-reloading",
15463        )
15464        .await;
15465        // The module sets the origin bit on its own rejection too: neither
15466        // the bit nor the daemon code may survive the relay as daemon-owned.
15467        let rejection = Frame::build(
15468            FrameType::Error,
15469            crate::router::daemon_error_flags(),
15470            0,
15471            0,
15472            bind.header.corr,
15473            serde_json::to_vec(&ErrorBody::new(
15474                "module_reloading",
15475                "module refused route.bind",
15476            ))
15477            .unwrap(),
15478        )
15479        .unwrap();
15480        handler
15481            .handle_control_frame(&module_ctx, rejection)
15482            .await
15483            .unwrap();
15484
15485        let response = route_task.await.unwrap();
15486        assert_eq!(response.len(), 1);
15487        assert_eq!(response[0].header.ty, FrameType::Error);
15488        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15489        assert!(
15490            !response[0].header.flags.is_daemon_origin(),
15491            "a module's own rejection must not carry daemon origin: {:?}",
15492            response[0].header.flags
15493        );
15494    }
15495
15496    #[tokio::test(flavor = "current_thread")]
15497    async fn route_open_module_rejection_uses_daemon_counter_key() {
15498        let registry = Arc::new(Registry::default());
15499        let forwarding = Arc::new(ForwardingTable::default());
15500        let handler =
15501            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15502        let module_connection = ConnectionId::new(95);
15503        let (module_ctx, mut module_rx) = route_ctx(module_connection);
15504        hello_via_sink(
15505            &handler,
15506            &module_ctx,
15507            &mut module_rx,
15508            hello_frame("aft", PROTOCOL_VERSION, 395),
15509        )
15510        .await;
15511
15512        let client_connection = ConnectionId::new(96);
15513        let (client_ctx, _client_rx) = route_ctx(client_connection);
15514        let capture = EventCapture::default();
15515        let _subscriber =
15516            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15517        let (route_task, bind) = relay_route_open(
15518            &handler,
15519            client_connection,
15520            &client_ctx.egress,
15521            &mut module_rx,
15522            396,
15523            "aft",
15524            "hostile-module-code",
15525        )
15526        .await;
15527        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15528        let rejection = Frame::build(
15529            FrameType::Error,
15530            control_flags(),
15531            0,
15532            0,
15533            bind.header.corr,
15534            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15535        )
15536        .unwrap();
15537        handler
15538            .handle_control_frame(&module_ctx, rejection)
15539            .await
15540            .unwrap();
15541
15542        let response = route_task.await.unwrap();
15543        assert_eq!(parse_error(&response[0])["code"], hostile_code);
15544        let counters = handler.counters().snapshot();
15545        assert_eq!(
15546            counters["route_open_refused_by_code"],
15547            json!({ "module_rejected": 1 })
15548        );
15549        assert!(counters["route_open_refused_by_code"]
15550            .get(hostile_code)
15551            .is_none());
15552
15553        let event = capture
15554            .events()
15555            .into_iter()
15556            .find(|event| {
15557                event.target == "control"
15558                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15559            })
15560            .expect("route.open module-rejection refusal event");
15561        let logged = event.fields.get("module_code").expect("module_code field");
15562        assert!(!logged.bytes().any(|byte| byte < 0x20));
15563        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15564    }
15565
15566    #[tokio::test]
15567    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15568        let registry = Arc::new(Registry::default());
15569        let supervisor_handle = SupervisorHandle::new();
15570        let missing_program = std::env::temp_dir().join(format!(
15571            "subc-route-open-missing-program-{}",
15572            std::process::id()
15573        ));
15574        let supervisor =
15575            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
15576                .with_handle(supervisor_handle.clone());
15577        let module = supervisor
15578            .supervise_configured(
15579                ModuleSpec {
15580                    carry_containment: false,
15581                    module_id: "failed".to_string(),
15582                    program: missing_program,
15583                    args: Vec::new(),
15584                    env: Vec::new(),
15585                    reserved: false,
15586                    reserved_prefixes: Vec::new(),
15587                    protocol: ModuleProtocol::Subc,
15588                    overlap: Default::default(),
15589                },
15590                true,
15591            )
15592            .unwrap();
15593        assert_eq!(module.state().unwrap(), ModuleState::Failed);
15594
15595        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15596        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15597        let response = handler
15598            .handle_control_frame(
15599                &ctx,
15600                route_open_frame(305, "failed", unique_project_root("failed")),
15601            )
15602            .await
15603            .unwrap();
15604
15605        assert_eq!(response[0].header.ty, FrameType::Error);
15606        let error = parse_error(&response[0]);
15607        assert_eq!(error["code"], "target_unavailable");
15608        assert!(error["message"]
15609            .as_str()
15610            .unwrap()
15611            .contains("state=failed, enabled=true, live=false"));
15612    }
15613
15614    #[tokio::test]
15615    async fn route_open_role_mismatch_remains_target_unavailable() {
15616        let registry = Arc::new(Registry::default());
15617        let handler = ControlHandler::new(Arc::clone(&registry));
15618        handler
15619            .handle_control(
15620                ConnectionId::new(41),
15621                non_routable_hello_frame_with_control_ops("health-only", 306, None),
15622            )
15623            .unwrap();
15624
15625        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15626        let response = handler
15627            .handle_control_frame(
15628                &ctx,
15629                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15630            )
15631            .await
15632            .unwrap();
15633
15634        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15635        assert!(parse_error(&response[0])["message"]
15636            .as_str()
15637            .unwrap()
15638            .contains("does not provide the requested target"));
15639    }
15640
15641    #[tokio::test]
15642    async fn route_open_inactive_registration_remains_target_unavailable() {
15643        let registry = Arc::new(Registry::default());
15644        let handler = ControlHandler::new(Arc::clone(&registry));
15645        handler
15646            .handle_control(
15647                ConnectionId::new(43),
15648                hello_frame("inactive", PROTOCOL_VERSION, 308),
15649            )
15650            .unwrap();
15651        assert!(registry
15652            .set_module_state_for_test("inactive", ChannelState::Closed)
15653            .unwrap());
15654
15655        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15656        let response = handler
15657            .handle_control_frame(
15658                &ctx,
15659                route_open_frame(309, "inactive", unique_project_root("inactive")),
15660            )
15661            .await
15662            .unwrap();
15663
15664        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15665        assert!(parse_error(&response[0])["message"]
15666            .as_str()
15667            .unwrap()
15668            .contains("is not active"));
15669    }
15670
15671    #[tokio::test]
15672    async fn late_health_reply_is_recorded_through_the_module_response_path() {
15673        let registry = Arc::new(Registry::default());
15674        let forwarding = Arc::new(ForwardingTable::default());
15675        let supervisor_handle = SupervisorHandle::new();
15676        let supervisor =
15677            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
15678                .with_forwarding(Arc::clone(&forwarding))
15679                .with_handle(supervisor_handle.clone());
15680        let module = supervisor
15681            .supervise_configured(
15682                crate::ModuleSpec {
15683                    carry_containment: false,
15684                    module_id: "late-health-response".to_string(),
15685                    program: PathBuf::from("disabled-module"),
15686                    args: Vec::new(),
15687                    env: Vec::new(),
15688                    reserved: false,
15689                    reserved_prefixes: Vec::new(),
15690                    protocol: ModuleProtocol::Subc,
15691                    overlap: Default::default(),
15692                },
15693                false,
15694            )
15695            .unwrap();
15696        let handler =
15697            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15698                .with_supervisor(supervisor_handle);
15699        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15700        handler
15701            .handle_control_frame(
15702                &module_ctx,
15703                hello_frame_with_control_ops(
15704                    "late-health-response",
15705                    PROTOCOL_VERSION,
15706                    7,
15707                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15708                ),
15709            )
15710            .await
15711            .unwrap();
15712        let probe_started_at = Instant::now() - Duration::from_millis(80);
15713        let pending = forwarding
15714            .begin_health_probe_rpc_for(
15715                "late-health-response",
15716                MODULE_CONTROL_OP_HEALTH_CHECK,
15717                probe_started_at,
15718                Instant::now() - Duration::from_millis(1),
15719            )
15720            .unwrap();
15721        assert!(forwarding
15722            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15723            .unwrap());
15724
15725        let responses = handler
15726            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15727            .await
15728            .unwrap();
15729
15730        assert!(responses.is_empty());
15731        let health = module.status().unwrap().health;
15732        assert_eq!(health.late_answer_count, 1);
15733        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15734    }
15735
15736    #[tokio::test]
15737    async fn health_probe_timeout_and_module_death_are_typed() {
15738        let registry = Arc::new(Registry::default());
15739        let forwarding = Arc::new(ForwardingTable::default());
15740        let handler =
15741            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15742                .with_health_probe_timeout(Duration::from_millis(50));
15743        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15744        hello_via_sink(
15745            &handler,
15746            &module_ctx,
15747            &mut module_rx,
15748            hello_frame_with_control_ops(
15749                "aft",
15750                PROTOCOL_VERSION,
15751                7,
15752                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15753            ),
15754        )
15755        .await;
15756
15757        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15758        let responses = handler
15759            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15760            .await
15761            .unwrap();
15762        assert_eq!(responses[0].header.ty, FrameType::Error);
15763        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15764        let _ = module_rx.try_recv();
15765
15766        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15767        let health_handler = handler.clone();
15768        let death_task = tokio::spawn(async move {
15769            health_handler
15770                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15771                .await
15772                .unwrap()
15773        });
15774        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15775            .await
15776            .unwrap()
15777            .unwrap();
15778        handler
15779            .cleanup_connection(module_ctx.connection_id)
15780            .unwrap();
15781        let responses = death_task.await.unwrap();
15782        assert_eq!(responses[0].header.ty, FrameType::Error);
15783        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15784    }
15785
15786    #[test]
15787    fn hello_requires_exact_protocol_version() {
15788        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15789            let registry = Arc::new(Registry::default());
15790            let handler = ControlHandler::new(Arc::clone(&registry));
15791            let responses = handler
15792                .handle_control(
15793                    ConnectionId::new(connection),
15794                    hello_frame("aft", offered, 9),
15795                )
15796                .unwrap();
15797
15798            assert_eq!(responses.len(), 1);
15799            assert_eq!(responses[0].header.ty, FrameType::Error);
15800            let error = parse_error(&responses[0]);
15801            assert_eq!(error["code"], "version_unsupported");
15802            assert!(registry.get_module("aft").unwrap().is_none());
15803            assert_eq!(registry.active_registration_count().unwrap(), 0);
15804        }
15805    }
15806
15807    #[test]
15808    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15809        let registry = Arc::new(Registry::default());
15810        let forwarding = Arc::new(ForwardingTable::default());
15811        let handler =
15812            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15813        let module_connection = ConnectionId::new(301);
15814        let registration = registry
15815            .register_with_control_ops(
15816                manifest("aft-push", PROTOCOL_VERSION),
15817                PROTOCOL_VERSION,
15818                module_connection,
15819                module_baseline_control_ops(),
15820            )
15821            .unwrap();
15822        let (module_tx, _module_rx) = mpsc::channel(8);
15823        let endpoint = forwarding
15824            .register_module_connection(
15825                module_connection,
15826                "aft-push".to_string(),
15827                PROTOCOL_VERSION,
15828                manifest_concurrency(&registration.manifest),
15829                FrameSink::new(module_tx),
15830            )
15831            .unwrap();
15832
15833        // A push op this version does not know is ignored (forward-compat), not errored.
15834        let unknown = Frame::build(
15835            FrameType::Push,
15836            control_flags(),
15837            0,
15838            0,
15839            5,
15840            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15841        )
15842        .unwrap();
15843        let out = handler.handle_status_update(endpoint, unknown).unwrap();
15844        assert!(
15845            out.is_empty(),
15846            "unknown push op must be ignored, got {out:?}"
15847        );
15848
15849        // A malformed body for a KNOWN op is a real error worth surfacing.
15850        let malformed = Frame::build(
15851            FrameType::Push,
15852            control_flags(),
15853            0,
15854            0,
15855            6,
15856            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15857        )
15858        .unwrap();
15859        let out = handler.handle_status_update(endpoint, malformed).unwrap();
15860        assert_eq!(out.len(), 1);
15861        assert_eq!(out[0].header.ty, FrameType::Error);
15862        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15863    }
15864
15865    #[test]
15866    fn hello_rejected_when_connection_already_owns_client_routes() {
15867        let registry = Arc::new(Registry::default());
15868        let forwarding = Arc::new(ForwardingTable::default());
15869        let handler =
15870            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15871        // Commits a client route on connection 202 (bound to a module on conn 101).
15872        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
15873        let client_connection = ConnectionId::new(202);
15874
15875        // That same connection now tries to register as a module: rejected, so one
15876        // connection never holds both client-route and module-endpoint state.
15877        let responses = handler
15878            .handle_control(
15879                client_connection,
15880                hello_frame("aft-second", PROTOCOL_VERSION, 9),
15881            )
15882            .unwrap();
15883        assert_eq!(responses[0].header.ty, FrameType::Error);
15884        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15885        assert!(registry.get_module("aft-second").unwrap().is_none());
15886    }
15887
15888    #[tokio::test]
15889    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15890        let registry = Arc::new(Registry::default());
15891        let forwarding = Arc::new(ForwardingTable::default());
15892        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15893        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15894        hello_via_sink(
15895            &handler,
15896            &module_ctx,
15897            &mut module_rx,
15898            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15899        )
15900        .await;
15901        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15902        let pending = forwarding
15903            .begin_route_bind_relay_for_test(
15904                client_ctx.connection_id,
15905                client_ctx.egress.clone(),
15906                2,
15907                "alpha",
15908            )
15909            .unwrap();
15910        forwarding
15911            .complete_pending_relay(
15912                module_ctx.connection_id,
15913                pending.corr,
15914                RouteBindRelayOutcome::Accepted,
15915            )
15916            .unwrap();
15917        client_rx.try_recv().unwrap();
15918        for module_id in ["beta", "alpha"] {
15919            let replies = handler
15920                .handle_control_frame(
15921                    &module_ctx,
15922                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15923                )
15924                .await
15925                .unwrap();
15926            assert_eq!(replies.len(), 1, "second HELLO must be refused");
15927            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15928        }
15929        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15930        assert!(registry.get_module("beta").unwrap().is_none());
15931        assert!(matches!(
15932            forwarding
15933                .lookup_data_route(
15934                    client_ctx.connection_id,
15935                    pending.client_channel,
15936                    pending.client_epoch,
15937                )
15938                .unwrap(),
15939            DataRoute::Client(DataRouteState::Bound(_))
15940        ));
15941        assert!(handler
15942            .hello_launch_nonces
15943            .lock()
15944            .unwrap()
15945            .presented(module_ctx.connection_id, Some("alpha-nonce")));
15946        assert!(module_rx.try_recv().is_err());
15947    }
15948
15949    #[test]
15950    fn reserved_module_hello_requires_matching_launch_nonce() {
15951        let registry = Arc::new(Registry::default());
15952        let supervisor = SupervisorHandle::new();
15953        // The supervisor recorded the nonce it injected when it spawned the reserved
15954        // module; the HELLO verifier checks against the same shared handle.
15955        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15956        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15957
15958        // A HELLO with NO nonce is rejected.
15959        let no_nonce = handler
15960            .handle_control(
15961                ConnectionId::new(1),
15962                hello_frame("vault", PROTOCOL_VERSION, 1),
15963            )
15964            .unwrap();
15965        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15966        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15967        assert!(registry.get_module("vault").unwrap().is_none());
15968
15969        // A HELLO with the WRONG nonce is rejected.
15970        let wrong = handler
15971            .handle_control(
15972                ConnectionId::new(2),
15973                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15974            )
15975            .unwrap();
15976        assert_eq!(wrong[0].header.ty, FrameType::Error);
15977        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15978        assert!(registry.get_module("vault").unwrap().is_none());
15979
15980        // A HELLO with the CORRECT nonce registers.
15981        let ok = handler
15982            .handle_control(
15983                ConnectionId::new(3),
15984                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15985            )
15986            .unwrap();
15987        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15988        assert!(registry.get_module("vault").unwrap().is_some());
15989    }
15990
15991    #[test]
15992    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15993        let registry = Arc::new(Registry::default());
15994        let supervisor = SupervisorHandle::new();
15995        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15996        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15997        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
15998
15999        let squat = handler
16000            .handle_control(
16001                ConnectionId::new(1),
16002                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
16003            )
16004            .unwrap();
16005        assert_eq!(squat[0].header.ty, FrameType::Error);
16006        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
16007        assert!(parse_error(&squat[0])["message"]
16008            .as_str()
16009            .unwrap()
16010            .contains("fed:"));
16011
16012        let accepted_peer = handler
16013            .handle_control(
16014                ConnectionId::new(2),
16015                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
16016            )
16017            .unwrap();
16018        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
16019
16020        let accepted_short = handler
16021            .handle_control(
16022                ConnectionId::new(3),
16023                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
16024            )
16025            .unwrap();
16026        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
16027
16028        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
16029            let response = handler
16030                .handle_control(
16031                    ConnectionId::new(conn),
16032                    hello_frame(module_id, PROTOCOL_VERSION, conn),
16033                )
16034                .unwrap();
16035            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
16036        }
16037    }
16038
16039    #[test]
16040    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
16041        let registry = Arc::new(Registry::default());
16042        let supervisor = SupervisorHandle::new();
16043        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
16044        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
16045        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
16046        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
16047
16048        let owner_nonce = handler
16049            .handle_control(
16050                ConnectionId::new(1),
16051                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
16052            )
16053            .unwrap();
16054        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
16055        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
16056        assert!(registry.get_module("fed:special").unwrap().is_none());
16057
16058        let exact_nonce = handler
16059            .handle_control(
16060                ConnectionId::new(2),
16061                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
16062            )
16063            .unwrap();
16064        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
16065        assert!(registry.get_module("fed:special").unwrap().is_some());
16066    }
16067
16068    #[test]
16069    fn non_reserved_module_ignores_launch_nonce() {
16070        let registry = Arc::new(Registry::default());
16071        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
16072        // registration succeeds whether a spawned process echoes a nonce or not.
16073        let handler = ControlHandler::new(Arc::clone(&registry));
16074        let no_nonce = handler
16075            .handle_control(
16076                ConnectionId::new(1),
16077                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
16078            )
16079            .unwrap();
16080        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
16081        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
16082
16083        let echoed_nonce = handler
16084            .handle_control(
16085                ConnectionId::new(2),
16086                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
16087            )
16088            .unwrap();
16089        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
16090        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
16091    }
16092
16093    #[test]
16094    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
16095        let handler = ControlHandler::default();
16096        let conn = ConnectionId::new(1);
16097        let malformed = Frame::build(
16098            FrameType::Hello,
16099            control_flags(),
16100            0,
16101            0,
16102            3,
16103            b"{not json".to_vec(),
16104        )
16105        .unwrap();
16106
16107        let error = handler.handle_control(conn, malformed).unwrap();
16108        assert_eq!(error[0].header.ty, FrameType::Error);
16109        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
16110
16111        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16112        let pong = handler.handle_control(conn, ping).unwrap();
16113        assert_eq!(pong[0].header.ty, FrameType::Pong);
16114        assert_eq!(pong[0].header.corr, 4);
16115    }
16116
16117    #[test]
16118    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
16119        let registry = Arc::new(Registry::default());
16120        let handler = ControlHandler::new(Arc::clone(&registry));
16121
16122        handler
16123            .handle_control(
16124                ConnectionId::new(1),
16125                hello_frame("aft", PROTOCOL_VERSION, 1),
16126            )
16127            .unwrap();
16128        let duplicate = handler
16129            .handle_control(
16130                ConnectionId::new(2),
16131                hello_frame("aft", PROTOCOL_VERSION, 2),
16132            )
16133            .unwrap();
16134
16135        assert_eq!(duplicate[0].header.ty, FrameType::Error);
16136        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16137        let registration = registry.get_module("aft").unwrap().unwrap();
16138        assert_eq!(registration.connection_id, ConnectionId::new(1));
16139    }
16140
16141    #[test]
16142    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
16143        let registry = Arc::new(Registry::default());
16144        let forwarding = Arc::new(ForwardingTable::default());
16145        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
16146        let handler =
16147            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16148                .with_process_liveness(process_liveness);
16149        let (ctx, route_channel, route_epoch) =
16150            bind_liveness_route(&registry, &forwarding, "aft-dead");
16151        let responses = handler
16152            .handle_route_poll(
16153                &ctx,
16154                route_poll_frame(41, PollKind::Liveness, route_channel),
16155                route_channel,
16156                route_epoch,
16157                PollKind::Liveness,
16158            )
16159            .unwrap();
16160
16161        assert_eq!(responses.len(), 1);
16162        assert_eq!(responses[0].header.ty, FrameType::Response);
16163        assert_route_poll_liveness(&responses[0], false);
16164    }
16165
16166    #[test]
16167    fn liveness_poll_without_process_source_uses_bound_route() {
16168        let registry = Arc::new(Registry::default());
16169        let forwarding = Arc::new(ForwardingTable::default());
16170        let handler =
16171            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
16172        let (ctx, route_channel, route_epoch) =
16173            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
16174        let responses = handler
16175            .handle_route_poll(
16176                &ctx,
16177                route_poll_frame(42, PollKind::Liveness, route_channel),
16178                route_channel,
16179                route_epoch,
16180                PollKind::Liveness,
16181            )
16182            .unwrap();
16183
16184        assert_route_poll_liveness(&responses[0], true);
16185    }
16186
16187    #[test]
16188    fn liveness_poll_untracked_process_source_uses_bound_route() {
16189        let registry = Arc::new(Registry::default());
16190        let forwarding = Arc::new(ForwardingTable::default());
16191        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
16192        let handler =
16193            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16194                .with_process_liveness(process_liveness);
16195        let (ctx, route_channel, route_epoch) =
16196            bind_liveness_route(&registry, &forwarding, "aft-untracked");
16197        let responses = handler
16198            .handle_route_poll(
16199                &ctx,
16200                route_poll_frame(43, PollKind::Liveness, route_channel),
16201                route_channel,
16202                route_epoch,
16203                PollKind::Liveness,
16204            )
16205            .unwrap();
16206
16207        assert_route_poll_liveness(&responses[0], true);
16208    }
16209
16210    #[tokio::test]
16211    async fn unknown_op_returns_unknown_control_op() {
16212        let handler = ControlHandler::default();
16213        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
16214        let request = Frame::build(
16215            FrameType::Request,
16216            control_flags(),
16217            0,
16218            0,
16219            55,
16220            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
16221        )
16222        .unwrap();
16223
16224        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16225
16226        assert_eq!(response.len(), 1);
16227        assert_eq!(response[0].header.ty, FrameType::Error);
16228        assert_eq!(response[0].header.corr, 55);
16229        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
16230    }
16231
16232    #[tokio::test]
16233    async fn supervisor_provenance_rejects_unknown_exact_module() {
16234        let handler = ControlHandler::default();
16235        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
16236        let request = Frame::build(
16237            FrameType::Request,
16238            control_flags(),
16239            0,
16240            0,
16241            57,
16242            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
16243        )
16244        .unwrap();
16245
16246        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16247
16248        assert_eq!(response.len(), 1);
16249        assert_eq!(response[0].header.ty, FrameType::Error);
16250        assert_eq!(response[0].header.corr, 57);
16251        let error = parse_error(&response[0]);
16252        assert_eq!(error["code"], "unknown_module");
16253        assert_eq!(error["message"], "module_id 'missing' is not supervised");
16254    }
16255
16256    #[test]
16257    fn provenance_probe_override_keeps_handler_tests_deterministic() {
16258        let expected = subc_control::RunningImageAgreement::Unavailable {
16259            reason: subc_control::RunningImageUnavailableReason::HashFailed,
16260        };
16261        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
16262        assert_eq!(handler.provenance_probe_override, Some(expected));
16263    }
16264
16265    #[test]
16266    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
16267        let verdict = reload_verdict(
16268            std::path::Path::new("/bin/new"),
16269            Some(std::path::Path::new("/bin/old")),
16270            subc_control::RunningImageAgreement::Unavailable {
16271                reason: subc_control::RunningImageUnavailableReason::HashFailed,
16272            },
16273        );
16274        assert!(matches!(
16275            verdict.path,
16276            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
16277                if configured == std::path::Path::new("/bin/new")
16278                    && spawned_from == std::path::Path::new("/bin/old")
16279        ));
16280    }
16281
16282    #[test]
16283    fn launch_spec_without_running_digest_is_unavailable_not_match() {
16284        use subc_control::{
16285            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
16286        };
16287        let configured = crate::supervise::ModuleSpec {
16288            carry_containment: false,
16289            module_id: "legacy".into(),
16290            program: "/module".into(),
16291            args: Vec::new(),
16292            env: Vec::new(),
16293            reserved: false,
16294            reserved_prefixes: Vec::new(),
16295            protocol: subc_control::ModuleProtocol::Subc,
16296            overlap: Default::default(),
16297        };
16298        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
16299        assert_eq!(
16300            launch_spec_agreement(&configured, false, Some(&digest)),
16301            Agreement::Unavailable {
16302                reason: Reason::NotRunning
16303            }
16304        );
16305        assert_eq!(
16306            launch_spec_agreement(&configured, true, None),
16307            Agreement::Unavailable {
16308                reason: Reason::SpawnedSpecUnavailable
16309            }
16310        );
16311        assert_eq!(
16312            launch_spec_agreement(&configured, true, Some(&digest)),
16313            Agreement::Match
16314        );
16315    }
16316
16317    #[cfg(windows)]
16318    #[test]
16319    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16320        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16321        let configured = dir.join("module.exe");
16322        std::fs::write(&configured, b"executable path fixture").unwrap();
16323        let spawned = std::fs::canonicalize(&configured).unwrap();
16324        assert_ne!(
16325            configured, spawned,
16326            "fixture must exercise different path spellings"
16327        );
16328        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16329        let lower_case = configured.to_string_lossy().to_lowercase();
16330        for spelling in [
16331            &configured,
16332            Path::new(&forward_slashes),
16333            Path::new(&lower_case),
16334        ] {
16335            let verdict = reload_verdict(
16336                spelling,
16337                Some(&spawned),
16338                subc_control::RunningImageAgreement::Unavailable {
16339                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16340                },
16341            );
16342            assert_eq!(
16343                verdict.path,
16344                ReloadPathAgreement::Match,
16345                "configured {spelling:?}, spawned {spawned:?}"
16346            );
16347        }
16348    }
16349
16350    #[cfg(windows)]
16351    #[test]
16352    fn windows_reload_path_keeps_different_existing_program_pending() {
16353        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16354        let old = dir.join("old.exe");
16355        let new = dir.join("new.exe");
16356        std::fs::write(&old, b"same executable content").unwrap();
16357        std::fs::copy(&old, &new).unwrap();
16358        let spawned = std::fs::canonicalize(&old).unwrap();
16359        let image = subc_control::RunningImageAgreement::Unavailable {
16360            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16361        };
16362        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16363        assert!(
16364            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16365            if configured == new && spawned_from == spawned)
16366        );
16367        let missing = dir.join("missing.exe");
16368        assert!(matches!(
16369            reload_verdict(&missing, Some(&spawned), image).path,
16370            ReloadPathAgreement::Mismatch { .. }
16371        ));
16372    }
16373
16374    #[test]
16375    fn reload_verdict_detects_replaced_image_at_same_path() {
16376        let image = subc_control::RunningImageAgreement::Mismatch {
16377            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16378                digest: "old".into(),
16379            },
16380            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16381                digest: "new".into(),
16382            },
16383        };
16384        let verdict = reload_verdict(
16385            std::path::Path::new("/bin/same"),
16386            Some(std::path::Path::new("/bin/same")),
16387            image.clone(),
16388        );
16389        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16390        assert_eq!(verdict.image, image);
16391    }
16392
16393    #[test]
16394    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16395        let image = subc_control::RunningImageAgreement::Unavailable {
16396            reason: subc_control::RunningImageUnavailableReason::NotRunning,
16397        };
16398        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16399        assert_eq!(
16400            verdict.path,
16401            subc_control::ReloadPathAgreement::Unavailable {
16402                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16403            }
16404        );
16405        assert_eq!(verdict.image, image);
16406
16407        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16408            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16409        };
16410        let verdict = reload_verdict(
16411            std::path::Path::new("/bin/same"),
16412            Some(std::path::Path::new("/bin/same")),
16413            unconfirmed.clone(),
16414        );
16415        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16416        assert_eq!(verdict.image, unconfirmed);
16417    }
16418
16419    #[test]
16420    fn reload_verdict_preserves_each_image_unavailability_reason() {
16421        use subc_control::RunningImageUnavailableReason as Reason;
16422
16423        for reason in [
16424            Reason::NotRunning,
16425            Reason::UnsupportedPlatform,
16426            Reason::RunningExecutableUnreadable,
16427            Reason::SpawnedPathUnreadable,
16428            Reason::HashFailed,
16429            Reason::ProcessIdentityUnconfirmed,
16430            Reason::Unknown("future_probe_reason".to_string()),
16431        ] {
16432            let image = subc_control::RunningImageAgreement::Unavailable {
16433                reason: reason.clone(),
16434            };
16435            let verdict = reload_verdict(
16436                std::path::Path::new("/bin/same"),
16437                Some(std::path::Path::new("/bin/same")),
16438                image.clone(),
16439            );
16440            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16441            assert_eq!(verdict.image, image, "{reason:?}");
16442        }
16443    }
16444
16445    #[tokio::test]
16446    async fn malformed_control_bodies_return_invalid_control_body() {
16447        let handler = ControlHandler::default();
16448        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16449
16450        for (corr, body) in [
16451            (56, br#"{"route_channel":1}"#.as_slice()),
16452            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16453            (
16454                58,
16455                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16456            ),
16457        ] {
16458            let request = Frame::build(
16459                FrameType::Request,
16460                control_flags(),
16461                0,
16462                0,
16463                corr,
16464                body.to_vec(),
16465            )
16466            .unwrap();
16467            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16468
16469            assert_eq!(response.len(), 1);
16470            assert_eq!(response[0].header.ty, FrameType::Error);
16471            assert_eq!(response[0].header.corr, corr);
16472            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16473        }
16474    }
16475
16476    #[tokio::test]
16477    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16478        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16479        let registry = Arc::new(Registry::default());
16480        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16481        let router = Router::with_control_handler(Arc::clone(&control));
16482        let connection = router.begin_connection();
16483        let (ctx, mut rx) = route_ctx(connection.id());
16484
16485        router
16486            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16487            .await
16488            .unwrap();
16489        let response = rx.recv().await.unwrap();
16490        let ack = parse_ack(&response);
16491        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16492        let channel = 1;
16493
16494        let goodbye =
16495            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16496        router.route_for_connection(&ctx, goodbye).await.unwrap();
16497        assert!(rx.try_recv().is_err());
16498        assert!(registry.get_module("aft").unwrap().is_none());
16499
16500        router
16501            .route_for_connection(&ctx, channel_request(channel, 13))
16502            .await
16503            .unwrap();
16504        let error_frame = rx.recv().await.unwrap();
16505        assert_eq!(error_frame.header.ty, FrameType::Error);
16506        assert_eq!(error_frame.header.channel, channel);
16507        let captured = crate::router::test_log::captured_logs(&logs);
16508        assert_eq!(
16509            captured
16510                .lines()
16511                .filter(|line| {
16512                    line.contains("module_id=aft")
16513                        && line.contains("reason=explicit_goodbye")
16514                        && line.contains("module registration ended")
16515                })
16516                .count(),
16517            1,
16518            "unexpected GOODBYE registry log: {captured}"
16519        );
16520    }
16521
16522    #[tokio::test]
16523    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16524        let registry = Arc::new(Registry::default());
16525        let handler = ControlHandler::new(registry).with_capability_config(
16526            [("prov".to_string(), true), ("cons".to_string(), true)],
16527            BTreeMap::new(),
16528        );
16529        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16530        register_capability_manifest(
16531            &handler,
16532            &provider_ctx,
16533            &mut provider_rx,
16534            capability_manifest("prov", &["thing/v1"], &[]),
16535            1,
16536        )
16537        .await;
16538        let mut consumer = capability_manifest("cons", &[], &[]);
16539        consumer.capabilities.as_mut().unwrap().requires.push(
16540            subc_protocol::manifest::CapabilityRequirement {
16541                capability: "thing/v1".to_string(),
16542                need: subc_protocol::manifest::CapabilityNeed::Required,
16543            },
16544        );
16545        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16546        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16547        assert_eq!(
16548            handler.capability_evaluator.verdict("cons", "thing/v1"),
16549            Some(CapabilityVerdict::Provided)
16550        );
16551        let (mut client_rx, _) = open_route_for_capability_test(
16552            &handler,
16553            &provider_ctx,
16554            &mut provider_rx,
16555            703,
16556            3,
16557            "prov",
16558            None,
16559        )
16560        .await;
16561        let goodbye =
16562            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16563        handler
16564            .handle_control_frame(&provider_ctx, goodbye)
16565            .await
16566            .unwrap();
16567        assert_eq!(
16568            handler.capability_evaluator.verdict("cons", "thing/v1"),
16569            Some(CapabilityVerdict::NeverProvided)
16570        );
16571        let closed = client_rx
16572            .try_recv()
16573            .expect("GOODBYE pushes route.closed before route GOODBYE");
16574        assert!(
16575            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16576            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16577        );
16578        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16579        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16580    }
16581
16582    struct RouteClosedFixture {
16583        handler: ControlHandler,
16584        #[cfg(unix)]
16585        supervisor: Supervisor,
16586        module_ctx: RouteCtx,
16587        #[cfg(unix)]
16588        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16589        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16590        route: Arc<crate::forwarding::RouteBinding>,
16591    }
16592
16593    async fn route_closed_fixture() -> RouteClosedFixture {
16594        let registry = Arc::new(Registry::default());
16595        let handler = ControlHandler::new(Arc::clone(&registry));
16596        #[cfg(unix)]
16597        let supervisor = Supervisor::new(registry, RestartPolicy::default())
16598            .with_forwarding(Arc::clone(&handler.forwarding));
16599        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16600        register_capability_manifest(
16601            &handler,
16602            &module_ctx,
16603            &mut module_rx,
16604            capability_manifest("prov", &[], &[]),
16605            1,
16606        )
16607        .await;
16608        let (client_rx, bind) = open_route_for_capability_test(
16609            &handler,
16610            &module_ctx,
16611            &mut module_rx,
16612            702,
16613            2,
16614            "prov",
16615            None,
16616        )
16617        .await;
16618        let ModuleControlRequest::RouteBind {
16619            route_channel: channel,
16620            epoch: route_epoch,
16621            ..
16622        } = bind
16623        else {
16624            panic!("expected route.bind");
16625        };
16626        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16627            handler
16628                .forwarding
16629                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16630                .unwrap()
16631        else {
16632            panic!("the acknowledged route must be bound");
16633        };
16634        RouteClosedFixture {
16635            handler,
16636            #[cfg(unix)]
16637            supervisor,
16638            module_ctx,
16639            #[cfg(unix)]
16640            module_rx,
16641            client_rx,
16642            route,
16643        }
16644    }
16645
16646    impl RouteClosedFixture {
16647        #[cfg(unix)]
16648        async fn cut(&mut self) {
16649            let notices = async {
16650                let mut draining = self.module_rx.recv().await.unwrap();
16651                assert!(matches!(
16652                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16653                        &draining.body
16654                    )
16655                    .unwrap(),
16656                    subc_protocol::session::ModuleControlCommand::Draining {
16657                        reason: RouteCloseReason::Disable,
16658                        ..
16659                    }
16660                ));
16661                draining.flushed.take().unwrap().send(()).unwrap();
16662                let mut closing = self.client_rx.recv().await.unwrap();
16663                assert!(matches!(
16664                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16665                    ClientControlPush::RouteClosing {
16666                        reason: RouteCloseReason::Restart,
16667                        ..
16668                    }
16669                ));
16670                closing.flushed.take().unwrap().send(()).unwrap();
16671            };
16672            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16673            result.unwrap();
16674        }
16675
16676        fn disconnect(&mut self) -> ClientControlPush {
16677            self.handler
16678                .cleanup_connection(self.module_ctx.connection_id)
16679                .unwrap();
16680            let closed = self.client_rx.try_recv().unwrap();
16681            assert_eq!(closed.header.ty, FrameType::Push);
16682            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16683            assert!(matches!(
16684                &push,
16685                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16686                    if channels == &[self.route.client_channel]
16687            ));
16688            assert_eq!(
16689                self.client_rx.try_recv().unwrap().header.ty,
16690                FrameType::Goodbye
16691            );
16692            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16693            push
16694        }
16695    }
16696
16697    #[cfg(unix)]
16698    #[tokio::test(start_paused = true)]
16699    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16700        let mut fixture = route_closed_fixture().await;
16701        fixture.cut().await;
16702        assert!(matches!(
16703            fixture.disconnect(),
16704            ClientControlPush::RouteClosed {
16705                reason: RouteCloseReason::Restart,
16706                drained: true,
16707                ..
16708            }
16709        ));
16710    }
16711
16712    #[cfg(unix)]
16713    #[tokio::test(start_paused = true)]
16714    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16715        let mut fixture = route_closed_fixture().await;
16716        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16717        let started = tokio::time::Instant::now();
16718        fixture.cut().await;
16719        assert!(started.elapsed() >= Duration::from_secs(2));
16720        assert_eq!(fixture.route.flow.in_flight(), 1);
16721        assert!(matches!(
16722            fixture.disconnect(),
16723            ClientControlPush::RouteClosed {
16724                reason: RouteCloseReason::Restart,
16725                drained: false,
16726                ..
16727            }
16728        ));
16729    }
16730
16731    #[tokio::test]
16732    async fn module_connection_crash_reports_crash_and_not_drained() {
16733        let mut fixture = route_closed_fixture().await;
16734        // The server calls cleanup_connection on module EOF. Without a planned
16735        // daemon shutdown, that disconnect must remain a crash with no prior
16736        // route.closing notice, even when the route has no in-flight calls.
16737        assert!(matches!(
16738            fixture.disconnect(),
16739            ClientControlPush::RouteClosed {
16740                reason: RouteCloseReason::Crash,
16741                drained: false,
16742                ..
16743            }
16744        ));
16745    }
16746
16747    #[tokio::test]
16748    async fn dropping_router_connection_releases_registration() {
16749        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16750        let registry = Arc::new(Registry::default());
16751        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
16752        let router = Router::with_control_handler(Arc::clone(&control));
16753        let connection = router.begin_connection();
16754        let connection_id = connection.id();
16755        let (ctx, mut rx) = route_ctx(connection_id);
16756
16757        router
16758            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16759            .await
16760            .unwrap();
16761        let response = rx.recv().await.unwrap();
16762        let ack = parse_ack(&response);
16763        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16764        assert!(registry.get_module("aft").unwrap().is_some());
16765
16766        drop(connection);
16767
16768        assert!(registry.get_module("aft").unwrap().is_none());
16769        assert_eq!(registry.active_registration_count().unwrap(), 0);
16770
16771        control
16772            .cleanup_connection(ConnectionId::new(u64::MAX))
16773            .unwrap();
16774        let captured = crate::router::test_log::captured_logs(&logs);
16775        println!("captured registry lifecycle logs:\n{captured}");
16776        let events: Vec<_> = captured
16777            .lines()
16778            .filter(|line| {
16779                line.contains("module registered module_id=aft ")
16780                    || line.contains("module registration ended")
16781            })
16782            .collect();
16783        assert_eq!(
16784            events.len(),
16785            2,
16786            "unexpected registry lifecycle logs: {captured}"
16787        );
16788        assert!(events[0].contains("module registered module_id=aft "));
16789        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16790        assert!(events[1].contains(&format!(
16791            "module_id=aft connection_id={}",
16792            connection_id.get()
16793        )));
16794        assert!(events[1].contains("reason=connection_closed"));
16795        assert!(events[1].contains("module registration ended"));
16796        assert_eq!(
16797            captured
16798                .lines()
16799                .filter(|line| line.contains("module registered module_id=aft "))
16800                .count(),
16801            1,
16802            "legacy registration admission line must appear once: {captured}"
16803        );
16804    }
16805
16806    #[tokio::test]
16807    async fn hello_registration_keeps_legacy_module_registered_line_once() {
16808        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16809        let registry = Arc::new(Registry::default());
16810        let control = ControlHandler::new(Arc::clone(&registry));
16811        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16812        hello_via_sink(
16813            &control,
16814            &ctx,
16815            &mut rx,
16816            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16817        )
16818        .await;
16819
16820        let captured = crate::router::test_log::captured_logs(&logs);
16821        let admissions: Vec<_> = captured
16822            .lines()
16823            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16824            .collect();
16825        assert_eq!(
16826            admissions.len(),
16827            1,
16828            "legacy admission line must remain exactly once: {captured}"
16829        );
16830        assert!(admissions[0].contains("routable_provider=true"));
16831        assert!(admissions[0].contains("connection_id=777"));
16832    }
16833
16834    fn capability_manifest(
16835        module_id: &str,
16836        provides: &[&str],
16837        must_never_reach: &[&str],
16838    ) -> ModuleManifest {
16839        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16840        manifest.capabilities = Some(CapabilityDeclarations {
16841            provides: provides
16842                .iter()
16843                .map(|capability| (*capability).to_string())
16844                .collect(),
16845            requires: Vec::new(),
16846            must_never_reach: must_never_reach
16847                .iter()
16848                .map(|capability| (*capability).to_string())
16849                .collect(),
16850        });
16851        manifest
16852    }
16853
16854    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16855        Frame::build(
16856            FrameType::Hello,
16857            control_flags(),
16858            0,
16859            0,
16860            corr,
16861            serde_json::to_vec(&ModuleHelloBody {
16862                protocol_ver: manifest.protocol_ver,
16863                manifest,
16864                control_ops: None,
16865                launch_nonce: None,
16866            })
16867            .expect("capability test HELLO serializes"),
16868        )
16869        .expect("capability test HELLO frame builds")
16870    }
16871
16872    fn catalog_update_with_capabilities_frame(
16873        corr: u64,
16874        capabilities: CapabilityDeclarations,
16875    ) -> Frame {
16876        Frame::build(
16877            FrameType::Request,
16878            control_flags(),
16879            0,
16880            0,
16881            corr,
16882            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16883                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16884                capabilities: Some(capabilities),
16885                ready: None,
16886            })
16887            .expect("capability catalog.update serializes"),
16888        )
16889        .expect("capability catalog.update frame builds")
16890    }
16891
16892    async fn register_capability_manifest(
16893        handler: &ControlHandler,
16894        ctx: &RouteCtx,
16895        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16896        manifest: ModuleManifest,
16897        corr: u64,
16898    ) {
16899        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16900    }
16901
16902    async fn open_route_for_capability_test(
16903        handler: &ControlHandler,
16904        target_ctx: &RouteCtx,
16905        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16906        client_connection_id: u64,
16907        corr: u64,
16908        target_module_id: &str,
16909        consumer_identity: Option<ConsumerIdentity>,
16910    ) -> (
16911        mpsc::Receiver<crate::router::OutboundFrame>,
16912        ModuleControlRequest,
16913    ) {
16914        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16915        let route_handler = handler.clone();
16916        let target_module_id = target_module_id.to_string();
16917        let route_task = tokio::spawn(async move {
16918            route_handler
16919                .handle_control_frame(
16920                    &client_ctx,
16921                    route_open_frame_with_admission_facts(
16922                        corr,
16923                        &target_module_id,
16924                        unique_project_root("admission-facts"),
16925                        consumer_identity,
16926                        None,
16927                    ),
16928                )
16929                .await
16930                .expect("capability test route.open succeeds")
16931        });
16932        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16933            .await
16934            .expect("capability test route.open must reach route.bind")
16935            .expect("target control receiver stays open");
16936        let bind_request: ModuleControlRequest =
16937            serde_json::from_slice(&bind.body).expect("route.bind decodes");
16938        handler
16939            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16940            .await
16941            .expect("capability test route.bind ACK succeeds");
16942        assert!(route_task.await.expect("route.open task joins").is_empty());
16943        let opened = client_rx
16944            .recv()
16945            .await
16946            .expect("successful route.open publishes a response");
16947        assert!(matches!(
16948            serde_json::from_slice::<ClientControlResponse>(&opened.body),
16949            Ok(ClientControlResponse::RouteOpen { .. })
16950        ));
16951        (client_rx, bind_request)
16952    }
16953
16954    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16955        assert_eq!(frame.header.ty, FrameType::Push);
16956        assert_eq!(frame.header.channel, 0);
16957        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16958            .expect("route.closed control push decodes");
16959        let ClientControlPush::RouteClosed { channels, .. } = &push else {
16960            panic!("expected route.closed");
16961        };
16962        assert_eq!(channels.len(), 1, "exactly one violating route closed");
16963        let closed_channel = channels[0];
16964        let channels = channels.clone();
16965        assert_eq!(
16966            push,
16967            ClientControlPush::RouteClosed {
16968                module_id: target_module_id.to_string(),
16969                channels,
16970                reason: RouteCloseReason::CapabilityDenied,
16971                drained: false,
16972                abandoned: 0,
16973                excluded_subscriptions: 0,
16974                terminal: Some(false),
16975            }
16976        );
16977        closed_channel
16978    }
16979
16980    #[tokio::test]
16981    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16982        let registry = Arc::new(Registry::default());
16983        let forwarding = Arc::new(ForwardingTable::default());
16984        let supervisor = SupervisorHandle::new();
16985        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16986        let handler =
16987            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16988                .with_supervisor(supervisor);
16989        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16990        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16991        register_capability_manifest(
16992            &handler,
16993            &target_ctx,
16994            &mut target_rx,
16995            capability_manifest("target", &["credentials-provider/v1"], &[]),
16996            1,
16997        )
16998        .await;
16999        register_capability_manifest(
17000            &handler,
17001            &opener_ctx,
17002            &mut opener_rx,
17003            capability_manifest("opener", &[], &["credentials-provider/v1"]),
17004            2,
17005        )
17006        .await;
17007
17008        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
17009        let replies = handler
17010            .handle_control_frame(
17011                &client_ctx,
17012                route_open_frame_with_admission_facts(
17013                    3,
17014                    "target",
17015                    unique_project_root("admission-facts"),
17016                    Some(ConsumerIdentity {
17017                        module_id: "opener".to_string(),
17018                        launch_nonce: "opener-nonce".to_string(),
17019                    }),
17020                    None,
17021                ),
17022            )
17023            .await
17024            .expect("denied route.open returns a typed frame");
17025        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17026        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17027        assert!(
17028            target_rx.try_recv().is_err(),
17029            "forbidden route.open must not relay route.bind"
17030        );
17031    }
17032
17033    #[tokio::test]
17034    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
17035        let registry = Arc::new(Registry::default());
17036        let forwarding = Arc::new(ForwardingTable::default());
17037        let supervisor = SupervisorHandle::new();
17038        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17039        let handler =
17040            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17041                .with_supervisor(supervisor);
17042        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
17043        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
17044        register_capability_manifest(
17045            &handler,
17046            &target_ctx,
17047            &mut target_rx,
17048            capability_manifest("target", &["credentials-provider/v1"], &[]),
17049            1,
17050        )
17051        .await;
17052        register_capability_manifest(
17053            &handler,
17054            &old_opener_ctx,
17055            &mut old_opener_rx,
17056            capability_manifest("opener", &[], &[]),
17057            2,
17058        )
17059        .await;
17060        let (mut client_rx, _) = open_route_for_capability_test(
17061            &handler,
17062            &target_ctx,
17063            &mut target_rx,
17064            712,
17065            3,
17066            "target",
17067            Some(ConsumerIdentity {
17068                module_id: "opener".to_string(),
17069                launch_nonce: "opener-nonce".to_string(),
17070            }),
17071        )
17072        .await;
17073        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17074
17075        handler
17076            .cleanup_connection(old_opener_ctx.connection_id)
17077            .expect("old opener registration cleans up");
17078        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
17079        register_capability_manifest(
17080            &handler,
17081            &new_opener_ctx,
17082            &mut new_opener_rx,
17083            capability_manifest("opener", &[], &["credentials-provider/v1"]),
17084            4,
17085        )
17086        .await;
17087
17088        let closed_channel = assert_capability_denied_push(
17089            client_rx
17090                .try_recv()
17091                .expect("HELLO deny addition must emit route.closed")
17092                .frame,
17093            "target",
17094        );
17095        let goodbye = client_rx
17096            .try_recv()
17097            .expect("HELLO deny addition must send the client a route GOODBYE")
17098            .frame;
17099        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17100        assert_eq!(goodbye.header.channel, closed_channel);
17101        assert_ne!(goodbye.header.channel, 0);
17102        assert_ne!(goodbye.header.epoch, 0);
17103        assert!(client_rx.try_recv().is_err());
17104        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17105        assert!(matches!(
17106            target_rx.try_recv(),
17107            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17108        ));
17109    }
17110
17111    #[tokio::test]
17112    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
17113        let registry = Arc::new(Registry::default());
17114        let forwarding = Arc::new(ForwardingTable::default());
17115        let supervisor = SupervisorHandle::new();
17116        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17117        let handler =
17118            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17119                .with_supervisor(supervisor);
17120        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
17121        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
17122        register_capability_manifest(
17123            &handler,
17124            &target_ctx,
17125            &mut target_rx,
17126            capability_manifest("target", &[], &[]),
17127            1,
17128        )
17129        .await;
17130        register_capability_manifest(
17131            &handler,
17132            &opener_ctx,
17133            &mut opener_rx,
17134            capability_manifest("opener", &[], &["credentials-provider/v1"]),
17135            2,
17136        )
17137        .await;
17138        let (mut client_rx, _) = open_route_for_capability_test(
17139            &handler,
17140            &target_ctx,
17141            &mut target_rx,
17142            722,
17143            3,
17144            "target",
17145            Some(ConsumerIdentity {
17146                module_id: "opener".to_string(),
17147                launch_nonce: "opener-nonce".to_string(),
17148            }),
17149        )
17150        .await;
17151        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17152
17153        let replies = handler
17154            .handle_control_frame(
17155                &target_ctx,
17156                catalog_update_with_capabilities_frame(
17157                    4,
17158                    CapabilityDeclarations {
17159                        provides: vec!["credentials-provider/v1".to_string()],
17160                        requires: Vec::new(),
17161                        must_never_reach: Vec::new(),
17162                    },
17163                ),
17164            )
17165            .await
17166            .expect("claim catalog.update succeeds");
17167        assert!(matches!(
17168            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
17169            Ok(ModuleControlResponseToModule::CatalogUpdate {})
17170        ));
17171        let closed_channel = assert_capability_denied_push(
17172            client_rx
17173                .try_recv()
17174                .expect("claim addition must emit route.closed")
17175                .frame,
17176            "target",
17177        );
17178        let goodbye = client_rx
17179            .try_recv()
17180            .expect("claim addition must send the client a route GOODBYE")
17181            .frame;
17182        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17183        assert_eq!(goodbye.header.channel, closed_channel);
17184        assert_ne!(goodbye.header.channel, 0);
17185        assert_ne!(goodbye.header.epoch, 0);
17186        assert!(client_rx.try_recv().is_err());
17187        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17188        assert!(matches!(
17189            target_rx.try_recv(),
17190            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17191        ));
17192    }
17193
17194    #[tokio::test]
17195    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
17196        let registry = Arc::new(Registry::default());
17197        let forwarding = Arc::new(ForwardingTable::default());
17198        let supervisor = SupervisorHandle::new();
17199        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17200        let handler =
17201            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17202                .with_supervisor(supervisor);
17203        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
17204        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
17205        register_capability_manifest(
17206            &handler,
17207            &target_ctx,
17208            &mut target_rx,
17209            capability_manifest("target", &["credentials-provider/v1"], &[]),
17210            1,
17211        )
17212        .await;
17213        register_capability_manifest(
17214            &handler,
17215            &opener_ctx,
17216            &mut opener_rx,
17217            capability_manifest("opener", &[], &[]),
17218            2,
17219        )
17220        .await;
17221        let (mut client_rx, _) = open_route_for_capability_test(
17222            &handler,
17223            &target_ctx,
17224            &mut target_rx,
17225            732,
17226            3,
17227            "target",
17228            Some(ConsumerIdentity {
17229                module_id: "opener".to_string(),
17230                launch_nonce: "opener-nonce".to_string(),
17231            }),
17232        )
17233        .await;
17234        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17235
17236        handler
17237            .handle_control_frame(
17238                &target_ctx,
17239                catalog_update_with_capabilities_frame(
17240                    4,
17241                    CapabilityDeclarations {
17242                        provides: Vec::new(),
17243                        requires: Vec::new(),
17244                        must_never_reach: Vec::new(),
17245                    },
17246                ),
17247            )
17248            .await
17249            .expect("claim removal catalog.update succeeds");
17250        assert_eq!(
17251            forwarding.active_binding_count().unwrap(),
17252            1,
17253            "removing an attested target claim must leave the route census unchanged"
17254        );
17255        assert!(
17256            client_rx.try_recv().is_err(),
17257            "claim removal must not emit route.closed or a client route GOODBYE"
17258        );
17259        assert!(
17260            target_rx.try_recv().is_err(),
17261            "claim removal must not send the target a route GOODBYE"
17262        );
17263    }
17264
17265    /// A direct client may open a route to a denied capability provider; this
17266    /// policy applies only to attested supervised module origins, not to direct clients.
17267    #[tokio::test]
17268    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
17269        let registry = Arc::new(Registry::default());
17270        let forwarding = Arc::new(ForwardingTable::default());
17271        let supervisor = SupervisorHandle::new();
17272        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17273        let handler =
17274            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17275                .with_supervisor(supervisor);
17276        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
17277        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
17278        register_capability_manifest(
17279            &handler,
17280            &target_ctx,
17281            &mut target_rx,
17282            capability_manifest("target", &["credentials-provider/v1"], &[]),
17283            1,
17284        )
17285        .await;
17286        register_capability_manifest(
17287            &handler,
17288            &opener_ctx,
17289            &mut opener_rx,
17290            capability_manifest("opener", &[], &["credentials-provider/v1"]),
17291            2,
17292        )
17293        .await;
17294
17295        let (_client_rx, bind) = open_route_for_capability_test(
17296            &handler,
17297            &target_ctx,
17298            &mut target_rx,
17299            742,
17300            3,
17301            "target",
17302            None,
17303        )
17304        .await;
17305        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17306            panic!("direct scope-honesty route must bind");
17307        };
17308        assert_eq!(principal, Some(Principal::Direct));
17309        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17310    }
17311
17312    /// A module that denies a capability receives no self-route exemption when it
17313    /// also attestedly provides that capability.
17314    #[tokio::test]
17315    async fn must_never_reach_self_route_is_capability_forbidden() {
17316        let registry = Arc::new(Registry::default());
17317        let forwarding = Arc::new(ForwardingTable::default());
17318        let supervisor = SupervisorHandle::new();
17319        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17320        let handler =
17321            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17322                .with_supervisor(supervisor);
17323        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17324        register_capability_manifest(
17325            &handler,
17326            &self_ctx,
17327            &mut self_rx,
17328            capability_manifest(
17329                "self-provider",
17330                &["credentials-provider/v1"],
17331                &["credentials-provider/v1"],
17332            ),
17333            1,
17334        )
17335        .await;
17336
17337        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17338        let replies = handler
17339            .handle_control_frame(
17340                &client_ctx,
17341                route_open_frame_with_admission_facts(
17342                    2,
17343                    "self-provider",
17344                    unique_project_root("admission-facts"),
17345                    Some(ConsumerIdentity {
17346                        module_id: "self-provider".to_string(),
17347                        launch_nonce: "self-nonce".to_string(),
17348                    }),
17349                    None,
17350                ),
17351            )
17352            .await
17353            .expect("self-route refusal returns a typed frame");
17354        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17355        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17356        assert!(
17357            self_rx.try_recv().is_err(),
17358            "self denial must not relay route.bind"
17359        );
17360    }
17361
17362    #[test]
17363    fn unsupported_channel_zero_frame_returns_error() {
17364        let handler = ControlHandler::default();
17365        let request = Frame::build(
17366            FrameType::Request,
17367            control_flags(),
17368            0,
17369            0,
17370            21,
17371            b"opaque".to_vec(),
17372        )
17373        .unwrap();
17374
17375        let response = handler
17376            .handle_control(ConnectionId::new(1), request)
17377            .unwrap();
17378
17379        assert_eq!(response[0].header.ty, FrameType::Error);
17380        assert_eq!(
17381            parse_error(&response[0])["code"],
17382            "unsupported_control_frame"
17383        );
17384    }
17385
17386    /// Blue/green swap at the control-plane boundary. The supervisor that opens
17387    /// a swap is not wired yet, so the candidate is registered here directly
17388    /// into the registry and forwarding candidate slots, the way the swap's
17389    /// HELLO admission will.
17390    mod swap {
17391        use super::*;
17392
17393        const INCUMBENT: ConnectionId = ConnectionId::new(30);
17394        const CANDIDATE: ConnectionId = ConnectionId::new(40);
17395
17396        struct Swap {
17397            registry: Arc<Registry>,
17398            forwarding: Arc<ForwardingTable>,
17399            handler: ControlHandler,
17400            incumbent_ctx: RouteCtx,
17401            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17402            candidate_ctx: RouteCtx,
17403            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17404        }
17405
17406        async fn swap_with_incumbent() -> Swap {
17407            let registry = Arc::new(Registry::default());
17408            let forwarding = Arc::new(ForwardingTable::default());
17409            let handler =
17410                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17411            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17412            hello_via_sink(
17413                &handler,
17414                &incumbent_ctx,
17415                &mut incumbent_rx,
17416                hello_frame("aft", PROTOCOL_VERSION, 7),
17417            )
17418            .await;
17419            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17420            Swap {
17421                registry,
17422                forwarding,
17423                handler,
17424                incumbent_ctx,
17425                incumbent_rx,
17426                candidate_ctx,
17427                candidate_rx,
17428            }
17429        }
17430
17431        fn register_candidate(swap: &Swap, ready: Option<bool>) {
17432            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17433            candidate_manifest.ready = ready;
17434            let registration = swap
17435                .registry
17436                .register_candidate_with_control_ops(
17437                    candidate_manifest,
17438                    PROTOCOL_VERSION,
17439                    CANDIDATE,
17440                    module_baseline_control_ops(),
17441                )
17442                .unwrap();
17443            swap.forwarding
17444                .register_candidate_module_connection(
17445                    CANDIDATE,
17446                    "aft".to_string(),
17447                    PROTOCOL_VERSION,
17448                    manifest_concurrency(&registration.manifest),
17449                    swap.candidate_ctx.egress.clone(),
17450                )
17451                .unwrap();
17452        }
17453
17454        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17455            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17456            swap.registry.promote_candidate("aft").unwrap().unwrap();
17457            cutover.incumbent.unwrap()
17458        }
17459
17460        /// A swap candidate's HELLO reserves its candidate registry entry
17461        /// before installing its candidate forwarding endpoint. The swap
17462        /// supervisor promotes a ready candidate as soon as it reads it as
17463        /// registered, and promotion cuts forwarding over to the candidate
17464        /// endpoint, so the lookup the supervisor reads
17465        /// (`Registry::published_registration`) must not return the candidate
17466        /// until that endpoint exists. Otherwise the cutover finds no candidate
17467        /// endpoint and the swap fails as if the candidate had disconnected.
17468        #[tokio::test]
17469        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17470            let mut swap = swap_with_incumbent().await;
17471            let hold = Arc::new(std::sync::Barrier::new(2));
17472            let handler = swap
17473                .handler
17474                .clone()
17475                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17476            handler
17477                .supervisor
17478                .open_swap("aft", "candidate-nonce".to_string());
17479            let hello = {
17480                let handler = handler.clone();
17481                let candidate_ctx = swap.candidate_ctx.clone();
17482                std::thread::spawn(move || {
17483                    handler
17484                        .handle_hello(
17485                            candidate_ctx.connection_id,
17486                            Some(candidate_ctx.egress.clone()),
17487                            hello_frame_with_nonce(
17488                                "aft",
17489                                PROTOCOL_VERSION,
17490                                8,
17491                                Some("candidate-nonce"),
17492                            ),
17493                        )
17494                        .unwrap()
17495                })
17496            };
17497
17498            hold.wait();
17499            // Parked: the candidate slot is claimed but not published, and
17500            // forwarding has no candidate endpoint.
17501            assert!(swap
17502                .registry
17503                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17504                .unwrap()
17505                .is_some());
17506            assert!(swap
17507                .registry
17508                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17509                .unwrap()
17510                .is_none());
17511            assert!(swap
17512                .forwarding
17513                .module_endpoint_for_connection(CANDIDATE)
17514                .unwrap()
17515                .is_none());
17516
17517            hold.wait();
17518            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17519            assert_eq!(
17520                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17521                FrameType::HelloAck
17522            );
17523            let published = swap
17524                .registry
17525                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17526                .unwrap()
17527                .expect("candidate is published once its HELLO completes");
17528            assert_eq!(published.connection_id, CANDIDATE);
17529            assert!(swap
17530                .forwarding
17531                .module_endpoint_for_connection(CANDIDATE)
17532                .unwrap()
17533                .is_some());
17534        }
17535
17536        fn keyed_total(counters: &Value, key: &str) -> u64 {
17537            counters[key]
17538                .as_object()
17539                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17540                .unwrap_or(0)
17541        }
17542
17543        #[tokio::test]
17544        async fn replacement_logs_old_end_and_new_admission_once() {
17545            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17546            let mut swap = swap_with_incumbent().await;
17547            swap.handler
17548                .supervisor
17549                .open_swap("aft", "candidate-nonce".to_string());
17550            hello_via_sink(
17551                &swap.handler,
17552                &swap.candidate_ctx,
17553                &mut swap.candidate_rx,
17554                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17555            )
17556            .await;
17557            cutover(&swap);
17558            swap.handler.cleanup_connection(INCUMBENT).unwrap();
17559
17560            let captured = crate::router::test_log::captured_logs(&logs);
17561            println!("captured replacement registry logs:\n{captured}");
17562            let new_admissions: Vec<_> = captured
17563                .lines()
17564                .filter(|line| {
17565                    line.contains("connection_id=40")
17566                        && line.contains("swap candidate registered; not routable until cutover")
17567                })
17568                .collect();
17569            assert_eq!(
17570                new_admissions.len(),
17571                1,
17572                "unexpected admission logs: {captured}"
17573            );
17574            assert!(new_admissions[0].contains("module_id=aft"));
17575            assert!(new_admissions[0].contains("connection_id=40"));
17576            assert!(
17577                new_admissions[0].contains("swap candidate registered; not routable until cutover")
17578            );
17579            assert!(new_admissions[0].contains("ready=true"));
17580
17581            let old_admissions: Vec<_> = captured
17582                .lines()
17583                .filter(|line| {
17584                    line.contains("module registered module_id=aft ")
17585                        && line.contains("connection_id=30")
17586                })
17587                .collect();
17588            assert_eq!(
17589                old_admissions.len(),
17590                1,
17591                "unexpected incumbent admission: {captured}"
17592            );
17593
17594            let old_ends: Vec<_> = captured
17595                .lines()
17596                .filter(|line| {
17597                    line.contains("connection_id=30") && line.contains("module registration ended")
17598                })
17599                .collect();
17600            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17601            assert!(old_ends[0].contains("module_id=aft"));
17602            assert!(old_ends[0].contains("connection_id=30"));
17603            assert!(old_ends[0].contains("reason=replaced"));
17604            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17605            assert_eq!(
17606                captured
17607                    .lines()
17608                    .filter(|line| line.contains("module registration promoted"))
17609                    .count(),
17610                1,
17611                "unexpected promotion logs: {captured}"
17612            );
17613        }
17614
17615        /// An ack from the incumbent for a bind it was sent before cutover,
17616        /// arriving before the incumbent is drained. The incumbent is the live
17617        /// connection carrying every other client's routes, so the ack must
17618        /// not end it: the waiting client is told to retry, the reservation is
17619        /// given back, and the incumbent is told to drop just that binding.
17620        #[tokio::test]
17621        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17622            let mut swap = swap_with_incumbent().await;
17623            let handler = swap.handler.clone();
17624
17625            // A co-tenant route, bound on the incumbent before the swap.
17626            let cotenant = ConnectionId::new(31);
17627            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17628            let (cotenant_task, cotenant_bind) = relay_route_open(
17629                &handler,
17630                cotenant,
17631                &cotenant_ctx.egress,
17632                &mut swap.incumbent_rx,
17633                100,
17634                "aft",
17635                "swap-cotenant",
17636            )
17637            .await;
17638            handler
17639                .handle_control_frame(
17640                    &swap.incumbent_ctx,
17641                    route_bind_ack(cotenant_bind.header.corr),
17642                )
17643                .await
17644                .unwrap();
17645            assert!(cotenant_task.await.unwrap().is_empty());
17646            let (cotenant_channel, cotenant_epoch) =
17647                published_route(&cotenant_rx.recv().await.unwrap());
17648
17649            // A second route.open, relayed to the incumbent and not yet acked.
17650            let caller = ConnectionId::new(32);
17651            let (caller_ctx, mut caller_rx) = route_ctx(caller);
17652            let (caller_task, caller_bind) = relay_route_open(
17653                &handler,
17654                caller,
17655                &caller_ctx.egress,
17656                &mut swap.incumbent_rx,
17657                101,
17658                "aft",
17659                "swap-caller",
17660            )
17661            .await;
17662            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17663
17664            register_candidate(&swap, None);
17665            cutover(&swap);
17666
17667            // The incumbent acks after promotion and before any drain.
17668            let ack = handler
17669                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17670                .await;
17671            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17672            if module_loop_error.is_some() {
17673                // What the connection loop does with an untranslated router
17674                // error: end the connection, releasing every route on it.
17675                handler.cleanup_connection(INCUMBENT).unwrap();
17676            }
17677
17678            // 1. The incumbent's other routes survive.
17679            assert!(
17680                cotenant_rx.try_recv().is_err(),
17681                "the co-tenant route on the incumbent was torn down by one late ack: \
17682                 {module_loop_error:?}"
17683            );
17684            assert!(matches!(
17685                swap.forwarding
17686                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17687                    .unwrap(),
17688                DataRoute::Client(DataRouteState::Bound(_))
17689            ));
17690            assert_eq!(module_loop_error, None);
17691            assert!(swap
17692                .registry
17693                .get_module_by_connection(INCUMBENT)
17694                .unwrap()
17695                .is_some());
17696
17697            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
17698            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17699                .await
17700                .expect("the incumbent is told to drop the abandoned binding")
17701                .unwrap()
17702                .frame;
17703            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17704            assert_eq!(goodbye.header.channel, abandoned_channel);
17705            assert_eq!(goodbye.header.epoch, abandoned_epoch);
17706            assert!(swap.incumbent_rx.try_recv().is_err());
17707
17708            // 3. The waiting client gets a retryable refusal and no route.
17709            let response = caller_task.await.unwrap();
17710            assert_eq!(response.len(), 1);
17711            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17712            assert!(caller_rx.try_recv().is_err());
17713
17714            // 4. The reservation pair is given back, and the pending bind
17715            //    settled exactly once: one accepted open (the co-tenant) and one
17716            //    refused open (the caller), nothing counted twice.
17717            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17718            let counters = handler.counters().snapshot();
17719            assert_eq!(
17720                keyed_total(&counters, "route_open_accepted_by_principal"),
17721                1
17722            );
17723            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17724            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17725        }
17726
17727        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
17728        /// id would resolve to the promoted candidate and every new route.open
17729        /// would be refused as reloading, leaving neither process routable.
17730        #[tokio::test]
17731        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17732            let mut swap = swap_with_incumbent().await;
17733            register_candidate(&swap, None);
17734            let incumbent = cutover(&swap);
17735            swap.forwarding
17736                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17737                .unwrap()
17738                .expect("the incumbent is still registered");
17739
17740            let client = ConnectionId::new(33);
17741            let (client_ctx, mut client_rx) = route_ctx(client);
17742            let route_handler = swap.handler.clone();
17743            let open_ctx = RouteCtx {
17744                connection_id: client,
17745                egress: client_ctx.egress.clone(),
17746            };
17747            let mut route_task = tokio::spawn(async move {
17748                route_handler
17749                    .handle_control_frame(
17750                        &open_ctx,
17751                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17752                    )
17753                    .await
17754                    .unwrap()
17755            });
17756            let bind = tokio::select! {
17757                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17758                response = &mut route_task => {
17759                    let response = response.unwrap();
17760                    panic!(
17761                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
17762                        parse_error(&response[0])["code"]
17763                    );
17764                }
17765            };
17766            swap.handler
17767                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17768                .await
17769                .unwrap();
17770            assert!(route_task.await.unwrap().is_empty());
17771            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17772            match swap
17773                .forwarding
17774                .lookup_data_route(client, channel, epoch)
17775                .unwrap()
17776            {
17777                DataRoute::Client(DataRouteState::Bound(route)) => {
17778                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17779                }
17780                other => panic!("expected a bound route on the candidate, got {other:?}"),
17781            }
17782            assert!(swap.incumbent_rx.try_recv().is_err());
17783        }
17784
17785        /// A candidate declares itself ready with `catalog.update` on its own
17786        /// connection. If the connection-keyed registry lookups searched only the
17787        /// active slot, this would answer `not_registered` and the candidate
17788        /// would never become ready.
17789        #[tokio::test]
17790        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17791            let swap = swap_with_incumbent().await;
17792            register_candidate(&swap, Some(false));
17793            let update = Frame::build(
17794                FrameType::Request,
17795                control_flags(),
17796                0,
17797                0,
17798                55,
17799                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17800                    provides: manifest("aft", PROTOCOL_VERSION).provides,
17801                    capabilities: None,
17802                    ready: Some(true),
17803                })
17804                .unwrap(),
17805            )
17806            .unwrap();
17807
17808            let replies = swap
17809                .handler
17810                .handle_control_frame(&swap.candidate_ctx, update)
17811                .await
17812                .unwrap();
17813
17814            assert_eq!(replies.len(), 1);
17815            assert_eq!(
17816                replies[0].header.ty,
17817                FrameType::Response,
17818                "candidate catalog.update was refused: {:?}",
17819                serde_json::from_slice::<Value>(&replies[0].body).ok()
17820            );
17821            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17822            assert_eq!(
17823                swap.registry
17824                    .get_module("aft")
17825                    .unwrap()
17826                    .unwrap()
17827                    .connection_id,
17828                INCUMBENT
17829            );
17830        }
17831    }
17832
17833    /// The HELLO gate while the supervisor has a swap open: only the nonce it
17834    /// minted for the candidate admits a second process, into the candidate
17835    /// slot, and that check runs ahead of the reserved-module gate.
17836    mod swap_admission {
17837        use super::*;
17838
17839        const INCUMBENT_NONCE: &str = "incumbent-nonce";
17840        const CANDIDATE_NONCE: &str = "candidate-nonce";
17841
17842        fn handler_with_incumbent(
17843            module_id: &str,
17844            reserved: bool,
17845        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17846            let registry = Arc::new(Registry::default());
17847            let supervisor = SupervisorHandle::new();
17848            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17849            if reserved {
17850                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17851            }
17852            let handler =
17853                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
17854            let incumbent = handler
17855                .handle_control(
17856                    ConnectionId::new(1),
17857                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17858                )
17859                .unwrap();
17860            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17861            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17862            (registry, supervisor, handler)
17863        }
17864
17865        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
17866        /// admits every nonce, so while a swap is open the swap gate is the only
17867        /// thing between a key-holder and the candidate slot. A nonce the
17868        /// supervisor did not mint, or none at all, is refused, and neither the
17869        /// incumbent's registration nor the candidate slot moves.
17870        #[test]
17871        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17872            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17873
17874            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17875                let replies = handler
17876                    .handle_control(
17877                        ConnectionId::new(connection),
17878                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17879                    )
17880                    .unwrap();
17881                assert_eq!(replies[0].header.ty, FrameType::Error);
17882                assert_eq!(
17883                    parse_error(&replies[0])["code"],
17884                    "swap_token_invalid",
17885                    "nonce {nonce:?}"
17886                );
17887            }
17888            assert!(registry.get_candidate("aft").unwrap().is_none());
17889            assert_eq!(
17890                registry.get_module("aft").unwrap().unwrap().connection_id,
17891                ConnectionId::new(1)
17892            );
17893
17894            // Control: the minted token is admitted, into the candidate slot,
17895            // and only once.
17896            let admitted = handler
17897                .handle_control(
17898                    ConnectionId::new(4),
17899                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17900                )
17901                .unwrap();
17902            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17903            assert_eq!(
17904                registry
17905                    .get_candidate("aft")
17906                    .unwrap()
17907                    .unwrap()
17908                    .connection_id,
17909                ConnectionId::new(4)
17910            );
17911            assert_eq!(
17912                registry.get_module("aft").unwrap().unwrap().connection_id,
17913                ConnectionId::new(1),
17914                "the candidate must not take the active slot"
17915            );
17916            let replayed = handler
17917                .handle_control(
17918                    ConnectionId::new(5),
17919                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17920                )
17921                .unwrap();
17922            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17923
17924            // The case only this gate covers: the incumbent has died mid-swap,
17925            // so its duplicate refusal is gone too, and without the gate a
17926            // key-holder would take the id's ACTIVE slot.
17927            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17928            let squatter = handler
17929                .handle_control(
17930                    ConnectionId::new(6),
17931                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17932                )
17933                .unwrap();
17934            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17935            assert!(
17936                registry.get_module("aft").unwrap().is_none(),
17937                "a squatter took the active slot of an id being swapped"
17938            );
17939        }
17940
17941        /// Design mutation arm (iii). A reserved module's candidate presents a
17942        /// nonce the reserved gate has never seen (that gate holds the
17943        /// incumbent's), so the swap gate must run first or the candidate is
17944        /// refused `reserved_module` and a reserved module can never be swapped.
17945        #[test]
17946        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17947            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17948
17949            let replies = handler
17950                .handle_control(
17951                    ConnectionId::new(2),
17952                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17953                )
17954                .unwrap();
17955
17956            assert_eq!(
17957                replies[0].header.ty,
17958                FrameType::HelloAck,
17959                "reserved candidate refused: {:?}",
17960                serde_json::from_slice::<Value>(&replies[0].body).ok()
17961            );
17962            assert_eq!(
17963                registry
17964                    .get_candidate("vault")
17965                    .unwrap()
17966                    .unwrap()
17967                    .connection_id,
17968                ConnectionId::new(2)
17969            );
17970        }
17971
17972        /// With no swap open the gate is inert: the incumbent's reserved gate
17973        /// and duplicate refusal behave exactly as before.
17974        #[test]
17975        fn without_an_open_swap_the_ordinary_gates_decide() {
17976            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17977            supervisor.close_swap("vault");
17978
17979            let candidate = handler
17980                .handle_control(
17981                    ConnectionId::new(2),
17982                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17983                )
17984                .unwrap();
17985            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17986            let duplicate = handler
17987                .handle_control(
17988                    ConnectionId::new(3),
17989                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17990                )
17991                .unwrap();
17992            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17993            assert!(registry.get_candidate("vault").unwrap().is_none());
17994        }
17995    }
17996
17997    /// `scope.sync` and `scope.describe` through the real control handler: who
17998    /// may sync is decided by the registration and launch nonce of the module
17999    /// connection, never by the request body.
18000    mod scopes {
18001        mod idle_tests {
18002            use super::*;
18003            include!("scope_idle_tests.rs");
18004        }
18005        mod route_open_timings {
18006            use super::*;
18007            include!("route_open_timing_tests.rs");
18008        }
18009        mod route_open_refusal_log {
18010            use super::*;
18011            include!("route_open_refusal_log_tests.rs");
18012        }
18013        mod activity {
18014            include!("activity_scope_tests.rs");
18015        }
18016        mod delta {
18017            include!("scope_handler_delta_tests.rs");
18018        }
18019        mod call_modes {
18020            include!("scope_call_mode_tests.rs");
18021        }
18022        mod federated {
18023            include!("scope_federated_tests.rs");
18024        }
18025        mod authorize {
18026            include!("route_authorize_tests.rs");
18027        }
18028        #[tokio::test]
18029        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
18030            let rig = rig().await;
18031            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
18032                "s".into(),
18033                1,
18034                "tool".into(),
18035            ))
18036            .unwrap();
18037            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
18038            let (direct, _rx) = wide_ctx(999);
18039            for (ctx, expected) in [
18040                (&rig.owner, FrameType::Response),
18041                (&direct, FrameType::Error),
18042            ] {
18043                let frame =
18044                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
18045                        .unwrap();
18046                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
18047                assert_eq!(replies[0].header.ty, expected);
18048                if expected == FrameType::Response {
18049                    let reply: subc_control::ScopeServedByReply =
18050                        serde_json::from_slice(&replies[0].body).unwrap();
18051                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
18052                } else {
18053                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
18054                }
18055            }
18056            let reply = call(
18057                &rig.handler,
18058                &rig.owner,
18059                &ModuleControlRequestFromModule::ScopeDescribe {
18060                    owner: Principal::Reserved {
18061                        module_id: OWNER.into(),
18062                    },
18063                    scope_ref: "s".into(),
18064                },
18065            )
18066            .await;
18067            assert!(matches!(
18068                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
18069                ModuleControlResponseToModule::ScopeDescribe { .. }
18070            ));
18071        }
18072
18073        use subc_protocol::scope::{
18074            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
18075            ScopeStatus,
18076        };
18077
18078        use super::*;
18079
18080        const OWNER: &str = "prefrontal-core";
18081
18082        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
18083            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
18084        }
18085
18086        async fn call(
18087            handler: &ControlHandler,
18088            ctx: &RouteCtx,
18089            request: &ModuleControlRequestFromModule,
18090        ) -> Frame {
18091            let body = serde_json::to_vec(request).unwrap();
18092            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
18093            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
18094            assert_eq!(replies.len(), 1, "{replies:?}");
18095            replies.pop().unwrap()
18096        }
18097
18098        async fn sync(
18099            handler: &ControlHandler,
18100            ctx: &RouteCtx,
18101            generation: u64,
18102            scopes: Vec<ScopeRecord>,
18103        ) -> Result<ModuleControlResponseToModule, String> {
18104            let reply = call(
18105                handler,
18106                ctx,
18107                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
18108            )
18109            .await;
18110            match reply.header.ty {
18111                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
18112                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
18113            }
18114        }
18115
18116        async fn describe(
18117            handler: &ControlHandler,
18118            ctx: &RouteCtx,
18119            owner: &str,
18120            scope_ref: &str,
18121        ) -> ModuleControlResponseToModule {
18122            let reply = call(
18123                handler,
18124                ctx,
18125                &ModuleControlRequestFromModule::ScopeDescribe {
18126                    owner: Principal::Reserved {
18127                        module_id: owner.to_string(),
18128                    },
18129                    scope_ref: scope_ref.to_string(),
18130                },
18131            )
18132            .await;
18133            assert_eq!(
18134                reply.header.ty,
18135                FrameType::Response,
18136                "{:?}",
18137                parse_error(&reply)
18138            );
18139            serde_json::from_slice(&reply.body).unwrap()
18140        }
18141
18142        #[tokio::test]
18143        async fn scope_describe_never_echoes_bind_proof_or_hash() {
18144            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18145            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18146            let mut rig = rig().await;
18147            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18148                .await;
18149            let _bound = rig
18150                .bound(
18151                    Some(AFT),
18152                    PLEXUS,
18153                    Some(
18154                        rig.selector("s", Some(1))
18155                            .with_bind_proof(Some(PROOF.into())),
18156                    ),
18157                )
18158                .await;
18159            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
18160            let output = serde_json::to_string(&reply).unwrap();
18161            assert!(output.contains("\"status\":\"live\""));
18162            assert!(!output.contains(PROOF));
18163            assert!(!output.contains(HASH));
18164        }
18165
18166        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
18167        async fn module(
18168            handler: &ControlHandler,
18169            connection: u64,
18170            module_id: &str,
18171            nonce: Option<&str>,
18172        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18173            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
18174            hello_via_sink(
18175                handler,
18176                &ctx,
18177                &mut rx,
18178                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
18179            )
18180            .await;
18181            (ctx, rx)
18182        }
18183
18184        /// `direct` and every other client connection has no registration, so
18185        /// it can neither sync nor own a scope.
18186        #[tokio::test]
18187        async fn a_client_connection_cannot_sync_or_describe() {
18188            let handler = ControlHandler::new(Arc::new(Registry::default()));
18189            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
18190            for request in [
18191                ModuleControlRequestFromModule::ScopeSync {
18192                    generation: 1,
18193                    scopes: vec![head("s", 1)],
18194                },
18195                ModuleControlRequestFromModule::ScopeDescribe {
18196                    owner: Principal::Direct,
18197                    scope_ref: "s".to_string(),
18198                },
18199            ] {
18200                let reply = call(&handler, &ctx, &request).await;
18201                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
18202            }
18203            assert!(
18204                !handler
18205                    .scopes
18206                    .read()
18207                    .unwrap()
18208                    .describe(
18209                        &Principal::Reserved {
18210                            module_id: OWNER.to_string()
18211                        },
18212                        "s"
18213                    )
18214                    .owner_synced
18215            );
18216        }
18217
18218        /// A module the supervisor did not spawn registers without a launch
18219        /// nonce, so it is never an owner's current launch.
18220        #[tokio::test]
18221        async fn a_module_without_a_supervised_launch_cannot_sync() {
18222            let handler = ControlHandler::new(Arc::new(Registry::default()));
18223            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
18224            assert_eq!(
18225                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
18226                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18227            );
18228        }
18229
18230        #[tokio::test]
18231        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
18232            let supervisor = SupervisorHandle::new();
18233            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18234            let handler = ControlHandler::new(Arc::new(Registry::default()))
18235                .with_supervisor(supervisor.clone());
18236            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18237            sync(&handler, &incumbent, 1, vec![head("s", 1)])
18238                .await
18239                .expect("the current launch syncs");
18240
18241            // A swap candidate registers with the swap token and is refused
18242            // while the incumbent keeps syncing.
18243            supervisor.open_swap(OWNER, "n2".to_string());
18244            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
18245            assert_eq!(
18246                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18247                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18248            );
18249            sync(&handler, &incumbent, 2, vec![head("s", 1)])
18250                .await
18251                .expect("the serving owner syncs during the swap");
18252
18253            // The swap fails and is rolled back. The candidate never held sync
18254            // authority, and still cannot sync.
18255            supervisor.close_swap(OWNER);
18256            assert_eq!(
18257                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18258                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18259            );
18260            sync(&handler, &incumbent, 3, vec![head("s", 1)])
18261                .await
18262                .expect("the serving owner syncs after the rollback");
18263            handler.cleanup_connection(candidate.connection_id).unwrap();
18264
18265            // A swap that cuts over. Promotion records the candidate's nonce as
18266            // the module's spawn nonce, which is what `set_spawn_nonce` does
18267            // here; the promoted connection then takes authority at any
18268            // generation and the superseded incumbent is refused.
18269            supervisor.open_swap(OWNER, "n3".to_string());
18270            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
18271            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
18272            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
18273                .await
18274                .expect("the promoted launch takes authority");
18275            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18276                panic!("unexpected reply {reply:?}");
18277            };
18278            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
18279            assert_eq!(
18280                sync(&handler, &incumbent, 4, Vec::new()).await,
18281                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18282            );
18283        }
18284
18285        /// Authority dies with its connection: the cleanup path releases it,
18286        /// so the owner's next connection takes it at any generation.
18287        #[tokio::test]
18288        async fn closing_the_authority_connection_frees_sync_authority() {
18289            let supervisor = SupervisorHandle::new();
18290            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18291            let handler = ControlHandler::new(Arc::new(Registry::default()))
18292                .with_supervisor(supervisor.clone());
18293            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18294            sync(&handler, &first, 10, vec![head("s", 1)])
18295                .await
18296                .unwrap();
18297            handler.cleanup_connection(first.connection_id).unwrap();
18298
18299            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
18300            sync(&handler, &second, 1, vec![head("s", 1)])
18301                .await
18302                .expect("the next connection takes the released authority");
18303        }
18304
18305        #[tokio::test]
18306        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18307            let supervisor = SupervisorHandle::new();
18308            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18309            let handler =
18310                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18311            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18312            sync(&handler, &first, 10, vec![head("s", 1)])
18313                .await
18314                .unwrap();
18315            handler
18316                .handle_control_frame(
18317                    &first,
18318                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18319                )
18320                .await
18321                .unwrap();
18322            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18323            sync(&handler, &second, 1, vec![head("s", 1)])
18324                .await
18325                .expect("GOODBYE releases authority even if the old socket remains open");
18326        }
18327
18328        #[tokio::test]
18329        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18330            let registry = Arc::new(Registry::default());
18331            let supervisor_handle = SupervisorHandle::new();
18332            let supervisor =
18333                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
18334                    .with_handle(supervisor_handle.clone())
18335                    .with_daemon_incarnation("incarnation-7".to_string());
18336            // Configured with enabled: false, so the supervisor lists the
18337            // module without spawning a process for it.
18338            supervisor
18339                .supervise_configured(
18340                    ModuleSpec {
18341                        carry_containment: false,
18342                        module_id: OWNER.to_string(),
18343                        program: PathBuf::from("/nonexistent/prefrontal-core"),
18344                        args: Vec::new(),
18345                        env: Vec::new(),
18346                        reserved: false,
18347                        reserved_prefixes: Vec::new(),
18348                        protocol: ModuleProtocol::Subc,
18349                        overlap: Default::default(),
18350                    },
18351                    false,
18352                )
18353                .unwrap();
18354            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18355            let handler =
18356                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
18357            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18358
18359            // Configured but not yet synced: a reader waits for the owner.
18360            let ModuleControlResponseToModule::ScopeDescribe {
18361                status,
18362                daemon_incarnation,
18363                owner_synced,
18364                owner_configured,
18365                scope,
18366                ..
18367            } = describe(&handler, &reader, OWNER, "s").await
18368            else {
18369                panic!("not a describe reply");
18370            };
18371            assert_eq!(status, ScopeStatus::NotLive);
18372            assert_eq!(daemon_incarnation, "incarnation-7");
18373            assert!(!owner_synced);
18374            assert!(owner_configured);
18375            assert!(scope.is_none());
18376
18377            // Not a supervised module: the owner will never sync, and a reader
18378            // refuses rather than waits.
18379            let ModuleControlResponseToModule::ScopeDescribe {
18380                status,
18381                owner_configured,
18382                ..
18383            } = describe(&handler, &reader, "ghost", "s").await
18384            else {
18385                panic!("not a describe reply");
18386            };
18387            assert_eq!(status, ScopeStatus::NotLive);
18388            assert!(!owner_configured);
18389
18390            // Live, with the stamp fields and the computed owner_authorized.
18391            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18392            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18393            let ModuleControlResponseToModule::ScopeDescribe {
18394                status,
18395                scope_epoch,
18396                owner_synced,
18397                scope,
18398                ..
18399            } = describe(&handler, &reader, OWNER, "s").await
18400            else {
18401                panic!("not a describe reply");
18402            };
18403            assert_eq!(status, ScopeStatus::Live);
18404            assert_eq!(scope_epoch, Some(4));
18405            assert!(owner_synced);
18406            let stamp = scope.expect("a live scope carries its stamp");
18407            assert!(
18408                stamp.owner_authorized,
18409                "prefrontal-core is the default authority"
18410            );
18411            assert_eq!(stamp.kind, ScopeKind::Head);
18412        }
18413
18414        #[tokio::test]
18415        async fn scope_authority_owners_decides_owner_authorized() {
18416            let supervisor = SupervisorHandle::new();
18417            supervisor.set_spawn_nonce("broca", "b1".to_string());
18418            let handler = ControlHandler::new(Arc::new(Registry::default()))
18419                .with_supervisor(supervisor)
18420                .with_scope_authority_owners(vec!["broca".to_string()]);
18421            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18422            let mut gated = head("s", 1);
18423            gated.attributes.agent_id = Some("agent".to_string());
18424            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18425            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18426                describe(&handler, &broca, "broca", "s").await
18427            else {
18428                panic!("not a describe reply");
18429            };
18430            assert!(scope.unwrap().owner_authorized);
18431        }
18432
18433        /// With route admission, the stamp, the commit re-check and drains in
18434        /// place, the feature is advertised: the module ops in HELLO_ACK, and
18435        /// `scopes/v1` in HELLO_ACK and `server.describe`.
18436        #[tokio::test]
18437        async fn scope_ops_and_the_scopes_capability_are_advertised() {
18438            let handler = ControlHandler::new(Arc::new(Registry::default()));
18439            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18440            let ack = hello_via_sink(
18441                &handler,
18442                &ctx,
18443                &mut rx,
18444                hello_frame("m", PROTOCOL_VERSION, 1),
18445            )
18446            .await;
18447            let ack = parse_ack(&ack);
18448            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18449                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18450            }
18451            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18452
18453            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18454            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18455            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18456            let reply = handler
18457                .handle_control_frame(&client, frame)
18458                .await
18459                .unwrap()
18460                .pop()
18461                .unwrap();
18462            let ClientControlResponse::ServerDescribe { capabilities, .. } =
18463                serde_json::from_slice(&reply.body).unwrap()
18464            else {
18465                panic!("not a server.describe reply");
18466            };
18467            assert!(
18468                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18469                "{capabilities:?}"
18470            );
18471        }
18472
18473        // ---- route admission, stamps, commit re-check and drains ----------
18474
18475        const PLEXUS: &str = "plexus";
18476        const OTHER: &str = "other";
18477        const AFT: &str = "aft";
18478        const BROCA: &str = "broca";
18479        const MAGIC: &str = "magic-context";
18480
18481        fn nonce(module_id: &str) -> String {
18482            format!("nonce-{module_id}")
18483        }
18484
18485        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18486            let (tx, rx) = mpsc::channel(64);
18487            (
18488                RouteCtx {
18489                    connection_id: ConnectionId::new(connection),
18490                    egress: FrameSink::new(tx),
18491                },
18492                rx,
18493            )
18494        }
18495
18496        /// A daemon with a configured owner (prefrontal-core) registered on its
18497        /// own module connection, two routable targets (plexus, other), and
18498        /// launch nonces minted for the modules that open routes as carriers.
18499        struct Rig {
18500            handler: ControlHandler,
18501            forwarding: Arc<ForwardingTable>,
18502            owner: RouteCtx,
18503            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18504            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18505            generation: u64,
18506            next_connection: u64,
18507            _supervisor: Supervisor,
18508        }
18509
18510        async fn rig() -> Rig {
18511            rig_with_flow_support(true).await
18512        }
18513
18514        async fn rig_with_flow_support(flow_support: bool) -> Rig {
18515            rig_with_scope_capabilities(flow_support, false).await
18516        }
18517
18518        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18519            let mut provides = Vec::new();
18520            if flow_support {
18521                provides.push("flow-scopes/v1");
18522            }
18523            if run_support {
18524                provides.push("agent-run-scopes/v1");
18525            }
18526            rig_with_target_capabilities(&provides).await
18527        }
18528
18529        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18530            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18531            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18532            // Decoder versions alone cannot admit scoped calls: only a capability
18533            // declaration promises the required run or flow behaviour.
18534            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18535            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18536            Frame::build(
18537                FrameType::Hello,
18538                control_flags(),
18539                0,
18540                0,
18541                connection,
18542                serde_json::to_vec(&body).unwrap(),
18543            )
18544            .unwrap()
18545        }
18546
18547        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18548            let registry = Arc::new(Registry::default());
18549            let forwarding = Arc::new(ForwardingTable::default());
18550            let supervisor_handle = SupervisorHandle::new();
18551            let supervisor =
18552                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
18553                    .with_handle(supervisor_handle.clone());
18554            supervisor
18555                .supervise_configured(
18556                    ModuleSpec {
18557                        carry_containment: false,
18558                        module_id: OWNER.to_string(),
18559                        program: PathBuf::from("/nonexistent/prefrontal-core"),
18560                        args: Vec::new(),
18561                        env: Vec::new(),
18562                        reserved: false,
18563                        reserved_prefixes: Vec::new(),
18564                        protocol: ModuleProtocol::Subc,
18565                        overlap: Default::default(),
18566                    },
18567                    false,
18568                )
18569                .unwrap();
18570            for module_id in [OWNER, AFT, BROCA, MAGIC] {
18571                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18572            }
18573            let handler =
18574                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18575                    .with_supervisor(supervisor_handle);
18576            let (owner, mut owner_rx) = wide_ctx(1);
18577            hello_via_sink(
18578                &handler,
18579                &owner,
18580                &mut owner_rx,
18581                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18582            )
18583            .await;
18584            let mut modules = BTreeMap::new();
18585            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18586                let (ctx, mut rx) = wide_ctx(connection);
18587                let hello = hello_with_capabilities(module_id, connection, provides);
18588                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18589                modules.insert(module_id.to_string(), (ctx, rx));
18590            }
18591            Rig {
18592                handler,
18593                forwarding,
18594                owner,
18595                _owner_rx: owner_rx,
18596                modules,
18597                generation: 0,
18598                next_connection: 100,
18599                _supervisor: supervisor,
18600            }
18601        }
18602
18603        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18604            ScopeCarrier::new(Principal::Reserved {
18605                module_id: module_id.to_string(),
18606            })
18607            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18608        }
18609
18610        /// The scope most tests open under: aft carries to any module, broca
18611        /// only to plexus and other, and the owner delegates as agent-1.
18612        fn session(scope_epoch: u64) -> ScopeRecord {
18613            let mut record = head("s", scope_epoch);
18614            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18615            record.attributes.agent_id = Some("agent-1".to_string());
18616            record.attributes.delegates = true;
18617            record
18618        }
18619
18620        impl Rig {
18621            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18622                self.generation += 1;
18623                sync(&self.handler, &self.owner, self.generation, scopes)
18624                    .await
18625                    .expect("the owner's sync is accepted");
18626            }
18627
18628            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18629                ScopeSelector::new(
18630                    Principal::Reserved {
18631                        module_id: OWNER.to_string(),
18632                    },
18633                    scope_ref.to_string(),
18634                    scope_epoch,
18635                )
18636            }
18637
18638            fn open_frame(
18639                &mut self,
18640                opener: Option<&str>,
18641                target: &str,
18642                scope: Option<ScopeSelector>,
18643            ) -> (
18644                RouteCtx,
18645                mpsc::Receiver<crate::router::OutboundFrame>,
18646                Frame,
18647            ) {
18648                self.next_connection += 1;
18649                let (ctx, rx) = wide_ctx(self.next_connection);
18650                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18651                    busy_refusals: None,
18652                    target: RouteTarget::ToolProvider {
18653                        module_id: target.to_string(),
18654                    },
18655                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18656                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
18657                        module_id: module_id.to_string(),
18658                        launch_nonce: nonce(module_id),
18659                    }),
18660                    consumer_capabilities: None,
18661                    role_versions: None,
18662                    admission_facts: None,
18663                    scope,
18664                })
18665                .unwrap();
18666                let frame = Frame::build(
18667                    FrameType::Request,
18668                    control_flags(),
18669                    0,
18670                    0,
18671                    self.next_connection,
18672                    body,
18673                )
18674                .unwrap();
18675                (ctx, rx, frame)
18676            }
18677
18678            /// Open and expect a refusal before anything is relayed.
18679            async fn refused(
18680                &mut self,
18681                opener: Option<&str>,
18682                target: &str,
18683                scope: Option<ScopeSelector>,
18684            ) -> String {
18685                self.refusal_body(opener, target, scope).await["code"]
18686                    .as_str()
18687                    .unwrap()
18688                    .to_string()
18689            }
18690
18691            async fn refusal_body(
18692                &mut self,
18693                opener: Option<&str>,
18694                target: &str,
18695                scope: Option<ScopeSelector>,
18696            ) -> Value {
18697                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18698                let replies = tokio::time::timeout(
18699                    Duration::from_secs(2),
18700                    self.handler.handle_control_frame(&ctx, frame),
18701                )
18702                .await
18703                .expect("the open must be refused before waiting for a bind ack")
18704                .unwrap();
18705                assert_eq!(replies.len(), 1, "{replies:?}");
18706                assert_eq!(replies[0].header.ty, FrameType::Error);
18707                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18708                assert!(
18709                    module_rx.try_recv().is_err(),
18710                    "a refused open relays nothing"
18711                );
18712                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18713                parse_error(&replies[0])
18714            }
18715
18716            /// Start an open and return its task and the bind the target got.
18717            async fn relayed(
18718                &mut self,
18719                opener: Option<&str>,
18720                target: &str,
18721                scope: Option<ScopeSelector>,
18722            ) -> Relayed {
18723                self.relayed_with_legacy_root(opener, target, scope, None)
18724                    .await
18725            }
18726
18727            async fn relayed_with_legacy_root(
18728                &mut self,
18729                opener: Option<&str>,
18730                target: &str,
18731                scope: Option<ScopeSelector>,
18732                legacy_root: Option<&std::path::Path>,
18733            ) -> Relayed {
18734                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18735                if let Some(root) = legacy_root {
18736                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18737                    body["identity"]["project_root"] = serde_json::json!(root);
18738                    frame.body = serde_json::to_vec(&body).unwrap();
18739                }
18740                let handler = self.handler.clone();
18741                let task_ctx = ctx.clone();
18742                let task = tokio::spawn(async move {
18743                    handler
18744                        .handle_control_frame(&task_ctx, frame)
18745                        .await
18746                        .unwrap()
18747                });
18748                let (_, module_rx) = self.modules.get_mut(target).unwrap();
18749                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18750                    .await
18751                    .expect("the target receives the relayed route.bind")
18752                    .unwrap()
18753                    .frame;
18754                Relayed {
18755                    target: target.to_string(),
18756                    client: ctx,
18757                    client_rx: rx,
18758                    task,
18759                    bind,
18760                }
18761            }
18762
18763            async fn ack(&self, relayed: &Relayed) {
18764                let (module, _) = &self.modules[&relayed.target];
18765                self.handler
18766                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18767                    .await
18768                    .unwrap();
18769            }
18770
18771            /// Open, ack and return the bound route.
18772            async fn bound(
18773                &mut self,
18774                opener: Option<&str>,
18775                target: &str,
18776                scope: Option<ScopeSelector>,
18777            ) -> Bound {
18778                let relayed = self.relayed(opener, target, scope).await;
18779                self.commit_relayed(relayed).await
18780            }
18781
18782            async fn bound_with_legacy_root(
18783                &mut self,
18784                opener: Option<&str>,
18785                target: &str,
18786                scope: Option<ScopeSelector>,
18787                root: &std::path::Path,
18788            ) -> Bound {
18789                let relayed = self
18790                    .relayed_with_legacy_root(opener, target, scope, Some(root))
18791                    .await;
18792                self.commit_relayed(relayed).await
18793            }
18794
18795            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18796                self.ack(&relayed).await;
18797                let Relayed {
18798                    target,
18799                    client,
18800                    mut client_rx,
18801                    task,
18802                    bind,
18803                } = relayed;
18804                assert!(
18805                    task.await.unwrap().is_empty(),
18806                    "the open is answered by commit"
18807                );
18808                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18809                Bound {
18810                    target,
18811                    client,
18812                    client_rx,
18813                    channel,
18814                    epoch,
18815                    bind,
18816                }
18817            }
18818
18819            fn live(&self, route: &Bound) -> bool {
18820                matches!(
18821                    self.forwarding
18822                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18823                        .unwrap(),
18824                    DataRoute::Client(DataRouteState::Bound(_))
18825                )
18826            }
18827        }
18828
18829        struct Relayed {
18830            target: String,
18831            client: RouteCtx,
18832            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18833            task: tokio::task::JoinHandle<Vec<Frame>>,
18834            bind: Frame,
18835        }
18836
18837        struct Bound {
18838            target: String,
18839            client: RouteCtx,
18840            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18841            channel: u16,
18842            epoch: u32,
18843            bind: Frame,
18844        }
18845
18846        impl Bound {
18847            /// The reason of the `route.closed` this client was sent, after
18848            /// checking it also got a GOODBYE on exactly this route.
18849            fn closed_reason(&mut self) -> RouteCloseReason {
18850                let mut reason = None;
18851                let mut goodbye = false;
18852                while let Ok(outbound) = self.client_rx.try_recv() {
18853                    let frame = outbound.frame;
18854                    match frame.header.ty {
18855                        FrameType::Goodbye => {
18856                            assert_eq!(
18857                                (frame.header.channel, frame.header.epoch),
18858                                (self.channel, self.epoch)
18859                            );
18860                            goodbye = true;
18861                        }
18862                        FrameType::Push => {
18863                            let ClientControlPush::RouteClosed {
18864                                reason: r,
18865                                module_id,
18866                                ..
18867                            } = serde_json::from_slice(&frame.body).unwrap()
18868                            else {
18869                                panic!("unexpected push");
18870                            };
18871                            assert_eq!(module_id, self.target);
18872                            reason = Some(r);
18873                        }
18874                        other => panic!("unexpected frame {other:?}"),
18875                    }
18876                }
18877                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18878                reason.expect("the client is told why the route closed")
18879            }
18880
18881            fn untouched(&mut self) -> bool {
18882                self.client_rx.try_recv().is_err()
18883            }
18884
18885            fn stamp(&self) -> Option<ScopeStamp> {
18886                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18887                    ModuleControlRequest::RouteBind { scope, .. } => scope,
18888                    other => panic!("expected a route.bind, got {other:?}"),
18889                }
18890            }
18891        }
18892
18893        #[tokio::test]
18894        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18895        ) {
18896            let mut rig = rig().await;
18897            let mut record = session(1);
18898            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18899            record.child_owners = vec![Principal::Reserved {
18900                module_id: MAGIC.to_string(),
18901            }];
18902            rig.sync(vec![record]).await;
18903            let scope = || Some(rig_selector("s", Some(1)));
18904
18905            // Admitted: the owner, a bare carrier to any module, a targeted
18906            // carrier to its listed module.
18907            rig.bound(Some(OWNER), PLEXUS, scope()).await;
18908            rig.bound(Some(AFT), OTHER, scope()).await;
18909            rig.bound(Some(BROCA), PLEXUS, scope()).await;
18910
18911            // Refused scope_not_carrier: a targeted carrier to an unlisted
18912            // module, a module that is not listed at all (a child owner is not
18913            // a carrier), and a direct key-holder.
18914            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18915                assert_eq!(
18916                    rig.refused(opener, target, scope()).await,
18917                    error_codes::SCOPE_NOT_CARRIER,
18918                    "{opener:?} -> {target}"
18919                );
18920            }
18921        }
18922
18923        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18924            ScopeSelector::new(
18925                Principal::Reserved {
18926                    module_id: OWNER.to_string(),
18927                },
18928                scope_ref.to_string(),
18929                scope_epoch,
18930            )
18931        }
18932
18933        #[tokio::test]
18934        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18935            let mut rig = rig().await;
18936            rig.sync(vec![session(1)]).await;
18937            for opener in [OWNER, AFT] {
18938                assert_eq!(
18939                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18940                        .await,
18941                    error_codes::SCOPE_EPOCH_REQUIRED,
18942                    "{opener}"
18943                );
18944            }
18945        }
18946
18947        #[tokio::test]
18948        async fn admission_separates_not_synced_not_live_and_ended() {
18949            let mut rig = rig().await;
18950            // Before the configured owner's first sync: retryable.
18951            let code = rig
18952                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18953                .await;
18954            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18955            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18956
18957            // An owner that is not configured will never sync: terminal.
18958            let ghost = ScopeSelector::new(
18959                Principal::Reserved {
18960                    module_id: "ghost".to_string(),
18961                },
18962                "s".to_string(),
18963                Some(1),
18964            );
18965            assert_eq!(
18966                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18967                error_codes::SCOPE_NOT_LIVE
18968            );
18969
18970            rig.sync(vec![session(2)]).await;
18971            assert_eq!(
18972                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18973                    .await,
18974                error_codes::SCOPE_NOT_LIVE
18975            );
18976            for epoch in [1, 3] {
18977                assert_eq!(
18978                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18979                        .await,
18980                    error_codes::SCOPE_ENDED,
18981                    "epoch {epoch}"
18982                );
18983            }
18984            // Control: the live epoch is admitted.
18985            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18986                .await;
18987        }
18988
18989        #[tokio::test]
18990        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18991            let mut rig = rig().await;
18992            rig.sync(vec![session(1)]).await;
18993            let route = rig
18994                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18995                .await;
18996            let stamp = route.stamp().expect("a scoped bind carries the stamp");
18997            assert_eq!(stamp.scope_ref, "s");
18998            assert_eq!(stamp.scope_epoch, 1);
18999            assert_eq!(stamp.kind, ScopeKind::Head);
19000            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
19001            assert!(stamp.attributes.delegates);
19002            assert!(stamp.owner_authorized);
19003            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19004            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
19005
19006            // broca owns a scope of its own on its own module connection; it is
19007            // not in scope_authority_owners, so its stamp is not authorized.
19008            let (broca, mut broca_rx) = wide_ctx(50);
19009            hello_via_sink(
19010                &rig.handler,
19011                &broca,
19012                &mut broca_rx,
19013                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
19014            )
19015            .await;
19016            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
19017                .await
19018                .unwrap();
19019            let own = ScopeSelector::new(
19020                Principal::Reserved {
19021                    module_id: BROCA.to_string(),
19022                },
19023                "b".to_string(),
19024                Some(1),
19025            );
19026            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
19027            assert!(!route.stamp().unwrap().owner_authorized);
19028        }
19029
19030        #[tokio::test]
19031        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
19032            let mut rig = rig().await;
19033            let mut record = head("s", 1);
19034            record.carriers = vec![carrier(AFT, None)];
19035            let flow_id = "Flow:run-7/step_2!~";
19036            record.attributes.flow_id = Some(flow_id.to_string());
19037            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
19038                .await
19039                .unwrap();
19040            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
19041                panic!("not a sync reply");
19042            };
19043            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
19044            let route = rig
19045                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19046                .await;
19047            assert!(rig.live(&route), "the stamped bind committed");
19048            let stamp = route.stamp().expect("a flow scope carries a stamp");
19049            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
19050            assert_eq!(stamp.attributes.agent_id, None);
19051            assert!(!stamp.attributes.delegates);
19052            assert!(stamp.owner_authorized);
19053            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19054            assert_eq!(unscoped.stamp(), None);
19055        }
19056
19057        #[tokio::test]
19058        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
19059            let mut rig = rig_with_flow_support(false).await;
19060            let mut record = session(1);
19061            record.attributes.flow_id = Some("flow:7".to_string());
19062            rig.sync(vec![record]).await;
19063            for opener in [OWNER, AFT] {
19064                let body = rig
19065                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19066                    .await;
19067                assert_eq!(body["code"], "target_flow_unsupported");
19068                let message = body["message"].as_str().unwrap();
19069                for required in [PLEXUS, "flow-scopes/v1"] {
19070                    assert!(message.contains(required), "{message}");
19071                }
19072            }
19073        }
19074
19075        #[tokio::test]
19076        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
19077            let mut rig = rig_with_flow_support(true).await;
19078            let mut record = session(1);
19079            record.attributes.flow_id = Some("flow:7".to_string());
19080            rig.sync(vec![record]).await;
19081            for opener in [OWNER, AFT] {
19082                let route = rig
19083                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19084                    .await;
19085                assert!(rig.live(&route));
19086                assert_eq!(
19087                    route.stamp().unwrap().attributes.flow_id.as_deref(),
19088                    Some("flow:7")
19089                );
19090            }
19091        }
19092
19093        #[tokio::test]
19094        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
19095            use std::future::Future;
19096
19097            let mut rig = rig_with_flow_support(true).await;
19098            let mut record = session(1);
19099            record.attributes.flow_id = Some("flow:7".to_string());
19100            rig.sync(vec![record]).await;
19101            let (client, mut client_rx, frame) =
19102                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
19103            // Hold the route-open response permit so admission sees the first
19104            // target but relay reservation cannot capture an endpoint yet.
19105            for _ in 0..64 {
19106                client.egress.try_send(route_bind_ack(1)).unwrap();
19107            }
19108            let handler = rig.handler.clone();
19109            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19110            std::future::poll_fn(|cx| {
19111                assert!(open.as_mut().poll(cx).is_pending());
19112                std::task::Poll::Ready(())
19113            })
19114            .await;
19115            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19116
19117            let old_connection = rig.modules[PLEXUS].0.connection_id;
19118            rig.handler.cleanup_connection(old_connection).unwrap();
19119            let (replacement, mut replacement_rx) = wide_ctx(200);
19120            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
19121            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
19122            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
19123            let hello = Frame::build(
19124                FrameType::Hello,
19125                control_flags(),
19126                0,
19127                0,
19128                200,
19129                serde_json::to_vec(&body).unwrap(),
19130            )
19131            .unwrap();
19132            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
19133
19134            client_rx.try_recv().unwrap();
19135            let replies = tokio::time::timeout(Duration::from_secs(2), open)
19136                .await
19137                .expect("the replacement is refused without waiting for a bind ack")
19138                .unwrap();
19139            assert_eq!(replies.len(), 1);
19140            let body = parse_error(&replies[0]);
19141            assert_eq!(body["code"], "target_flow_unsupported");
19142            for required in [PLEXUS, "flow-scopes/v1"] {
19143                assert!(body["message"].as_str().unwrap().contains(required));
19144            }
19145            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
19146            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19147            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19148            assert!(rig
19149                .handler
19150                .registry
19151                .get_module_by_connection(replacement.connection_id)
19152                .unwrap()
19153                .is_some());
19154        }
19155
19156        #[tokio::test]
19157        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
19158            use std::future::Future;
19159            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
19160            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19161            const NEXT_HASH: &str =
19162                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19163            let mut rig = rig().await;
19164            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
19165            record.carriers.push(ScopeCarrier::new(Principal::Direct));
19166            rig.sync(vec![record.clone()]).await;
19167            let (client, mut rx, frame) = rig.open_frame(
19168                None,
19169                PLEXUS,
19170                Some(
19171                    rig.selector("s", Some(1))
19172                        .with_bind_proof(Some(PROOF.into())),
19173                ),
19174            );
19175            // Hold egress reservation while the scope owner changes its bind-proof
19176            // hash, so a proof valid at admission must be checked again at relay.
19177            for _ in 0..64 {
19178                client.egress.try_send(route_bind_ack(1)).unwrap();
19179            }
19180            let handler = rig.handler.clone();
19181            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19182            std::future::poll_fn(|cx| {
19183                assert!(open.as_mut().poll(cx).is_pending());
19184                std::task::Poll::Ready(())
19185            })
19186            .await;
19187            record.bind_proof_sha256 = Some(NEXT_HASH.into());
19188            rig.sync(vec![record]).await;
19189            rx.try_recv().unwrap();
19190            let replies = tokio::time::timeout(Duration::from_secs(2), open)
19191                .await
19192                .unwrap()
19193                .unwrap();
19194            assert_eq!(
19195                parse_error(&replies[0])["code"],
19196                error_codes::SCOPE_BIND_PROOF_MISMATCH
19197            );
19198            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
19199            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19200            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19201        }
19202
19203        #[tokio::test]
19204        async fn scope_bind_proof_reserved_openers_ignore_proof() {
19205            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19206            let mut rig = rig().await;
19207            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19208                .await;
19209            for opener in [OWNER, AFT] {
19210                for proof in [None, Some("malformed+proof=")] {
19211                    let route = rig
19212                        .bound(
19213                            Some(opener),
19214                            PLEXUS,
19215                            Some(
19216                                rig.selector("s", Some(1))
19217                                    .with_bind_proof(proof.map(str::to_string)),
19218                            ),
19219                        )
19220                        .await;
19221                    assert!(rig.live(&route));
19222                }
19223            }
19224        }
19225
19226        #[tokio::test]
19227        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
19228            use std::future::Future;
19229            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19230            const NEXT_HASH: &str =
19231                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19232            let mut rig = rig().await;
19233            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19234                .await;
19235            let (client, mut rx, frame) =
19236                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
19237            for _ in 0..64 {
19238                client.egress.try_send(route_bind_ack(1)).unwrap();
19239            }
19240            let handler = rig.handler.clone();
19241            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19242            std::future::poll_fn(|cx| {
19243                assert!(open.as_mut().poll(cx).is_pending());
19244                std::task::Poll::Ready(())
19245            })
19246            .await;
19247            rig.sync(vec![
19248                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
19249            ])
19250            .await;
19251            rx.try_recv().unwrap();
19252            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19253            let bind = std::future::poll_fn(|cx| {
19254                assert!(open.as_mut().poll(cx).is_pending());
19255                module_rx.poll_recv(cx)
19256            })
19257            .await
19258            .unwrap()
19259            .frame;
19260            handler
19261                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
19262                .await
19263                .unwrap();
19264            let replies = tokio::time::timeout(Duration::from_secs(2), open)
19265                .await
19266                .unwrap()
19267                .unwrap();
19268            assert_eq!(replies.len(), 1);
19269            // Proof rechecking allowed the relay. Independent tag settlement
19270            // still asks the opener to retry after any same-epoch content change.
19271            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
19272            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19273            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
19274            let route = rig
19275                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
19276                .await;
19277            assert!(rig.live(&route));
19278        }
19279
19280        #[tokio::test]
19281        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
19282        {
19283            let mut rig = rig_with_flow_support(false).await;
19284            rig.sync(vec![session(1)]).await;
19285            let route = rig
19286                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19287                .await;
19288            assert!(rig.live(&route));
19289            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
19290            let mut record = session(1);
19291            record.attributes.flow_id = Some("flow:7".to_string());
19292            rig.sync(vec![record]).await;
19293            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
19294            assert!(rig.live(&unscoped));
19295            assert_eq!(unscoped.stamp(), None);
19296        }
19297
19298        #[tokio::test]
19299        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
19300            let mut rig = rig().await;
19301            let mut record = session(1);
19302            record.attributes.flow_id = Some("flow:7".to_string());
19303            rig.sync(vec![record.clone()]).await;
19304            let before = rig.forwarding.published_scope_tag(OWNER, "s");
19305            let mut owner_route = rig
19306                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19307                .await;
19308            let mut carrier_route = rig
19309                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19310                .await;
19311            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19312            rig.sync(vec![record.clone()]).await;
19313            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19314            assert!(rig.live(&owner_route) && owner_route.untouched());
19315            assert!(rig.live(&carrier_route) && carrier_route.untouched());
19316
19317            record.attributes.flow_id = Some("flow:8".to_string());
19318            rig.sync(vec![record]).await;
19319            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19320            let before = before.unwrap();
19321            assert_eq!(after.scope_epoch, before.scope_epoch);
19322            assert!(after.version > before.version);
19323            for route in [&mut owner_route, &mut carrier_route] {
19324                assert!(!rig.live(route));
19325                assert_eq!(
19326                    route.closed_reason(),
19327                    RouteCloseReason::ScopeDelegationChanged
19328                );
19329            }
19330            assert!(rig.live(&unscoped) && unscoped.untouched());
19331            // Each provider also receives a GOODBYE for its drained route;
19332            // consume it before expecting the next route.bind on that sink.
19333            for target in [PLEXUS, OTHER] {
19334                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19335                let goodbye = module_rx
19336                    .try_recv()
19337                    .expect("the provider sees the drain")
19338                    .frame;
19339                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19340                assert!(module_rx.try_recv().is_err());
19341            }
19342            let rebound = rig
19343                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19344                .await;
19345            assert_eq!(
19346                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19347                Some("flow:8")
19348            );
19349        }
19350
19351        /// The owner's sync lands between admission and the module's ack. The
19352        /// open is refused by name, the module's other routes stay up, and the
19353        /// reserved pair is released. Changed content is retryable; an ended
19354        /// scope is not.
19355        #[tokio::test]
19356        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19357            let mut rig = rig().await;
19358            rig.sync(vec![session(1)]).await;
19359            let mut cotenant = rig
19360                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19361                .await;
19362
19363            let mut changed = session(1);
19364            changed.child_owners.push(Principal::Reserved {
19365                module_id: MAGIC.to_string(),
19366            });
19367            let mut ended = None;
19368            for (code, next) in [
19369                (error_codes::SCOPE_CHANGED, vec![changed]),
19370                (error_codes::SCOPE_ENDED, Vec::new()),
19371            ] {
19372                let relayed = rig
19373                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19374                    .await;
19375                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19376                ended = Some(next.is_empty());
19377                rig.sync(next).await;
19378                rig.ack(&relayed).await;
19379                let replies = relayed.task.await.unwrap();
19380                assert_eq!(replies.len(), 1, "{replies:?}");
19381                assert_eq!(parse_error(&replies[0])["code"], code);
19382                assert_eq!(
19383                    subc_protocol::error_codes::is_retryable_route_open(code),
19384                    code == error_codes::SCOPE_CHANGED
19385                );
19386                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19387                // The module is told to drop just the binding it created.
19388                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19389                // Collected, because ending the scope also closes the co-tenant
19390                // route, whose GOODBYE comes first.
19391                let mut goodbyes = Vec::new();
19392                while let Ok(outbound) = plexus_rx.try_recv() {
19393                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19394                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19395                }
19396                assert!(
19397                    goodbyes.contains(&(bind_channel, bind_epoch)),
19398                    "{goodbyes:?}"
19399                );
19400                assert!(rig
19401                    .handler
19402                    .registry
19403                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19404                    .unwrap()
19405                    .is_some());
19406            }
19407            assert_eq!(ended, Some(true));
19408            // The co-tenant stayed up through the change, and closed only when
19409            // the scope ended, by the drain rule rather than by the commit.
19410            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19411        }
19412
19413        /// Each row of the drain table on one set of routes: the owner's, a
19414        /// bare carrier's, and a targeted carrier's to each of its targets.
19415        #[tokio::test]
19416        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19417            struct Case {
19418                name: &'static str,
19419                change: fn(&mut ScopeRecord),
19420                /// Closed routes by index: owner->plexus, aft->plexus,
19421                /// broca->plexus, broca->other.
19422                closed: [Option<RouteCloseReason>; 4],
19423            }
19424            use RouteCloseReason::*;
19425            let cases = [
19426                Case {
19427                    name: "a carrier entry removed",
19428                    change: |r| {
19429                        r.carriers.retain(|c| {
19430                            c.principal
19431                                != Principal::Reserved {
19432                                    module_id: AFT.to_string(),
19433                                }
19434                        })
19435                    },
19436                    closed: [None, Some(ScopeCarrierRemoved), None, None],
19437                },
19438                Case {
19439                    name: "a target removed from a carrier",
19440                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19441                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
19442                },
19443                Case {
19444                    name: "a bare carrier narrowed to targets",
19445                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19446                    closed: [None, Some(ScopeCarrierRemoved), None, None],
19447                },
19448                Case {
19449                    name: "delegates turned off",
19450                    change: |r| r.attributes.delegates = false,
19451                    closed: [Some(ScopeDelegationChanged); 4],
19452                },
19453                Case {
19454                    name: "agent_id changed",
19455                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19456                    closed: [Some(ScopeDelegationChanged); 4],
19457                },
19458                Case {
19459                    name: "a carrier added, child owners changed, the record re-sent",
19460                    change: |r| {
19461                        r.carriers.push(carrier(MAGIC, None));
19462                        r.child_owners.push(Principal::Reserved {
19463                            module_id: MAGIC.to_string(),
19464                        });
19465                    },
19466                    closed: [None; 4],
19467                },
19468                Case {
19469                    name: "a target added",
19470                    change: |r| {
19471                        r.carriers[1]
19472                            .targets
19473                            .as_mut()
19474                            .unwrap()
19475                            .push("third".to_string())
19476                    },
19477                    closed: [None; 4],
19478                },
19479                Case {
19480                    name: "delegates turned on",
19481                    change: |r| r.attributes.delegates = true,
19482                    closed: [None; 4],
19483                },
19484            ];
19485            for case in cases {
19486                let mut rig = rig().await;
19487                rig.sync(vec![session(1)]).await;
19488                let scope = || Some(rig_selector("s", Some(1)));
19489                let mut routes = [
19490                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
19491                    rig.bound(Some(AFT), PLEXUS, scope()).await,
19492                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
19493                    rig.bound(Some(BROCA), OTHER, scope()).await,
19494                ];
19495                let mut record = session(1);
19496                (case.change)(&mut record);
19497                rig.sync(vec![record]).await;
19498                for (index, expected) in case.closed.iter().enumerate() {
19499                    let route = &mut routes[index];
19500                    match expected {
19501                        Some(reason) => {
19502                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
19503                            assert_eq!(
19504                                route.closed_reason(),
19505                                *reason,
19506                                "{}: route {index}",
19507                                case.name
19508                            );
19509                        }
19510                        None => {
19511                            assert!(rig.live(route), "{}: route {index} closed", case.name);
19512                            assert!(
19513                                route.untouched(),
19514                                "{}: route {index} was told something",
19515                                case.name
19516                            );
19517                        }
19518                    }
19519                }
19520            }
19521        }
19522
19523        #[tokio::test]
19524        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19525            // Removed, and replaced by a higher epoch.
19526            for next in [Vec::new(), vec![session(2)]] {
19527                let mut rig = rig().await;
19528                rig.sync(vec![session(1)]).await;
19529                let mut route = rig
19530                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19531                    .await;
19532                rig.sync(next).await;
19533                assert!(!rig.live(&route));
19534                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19535            }
19536
19537            // A child whose parent ends: its routes close as parent-ended, the
19538            // child stays live, and routes under the parent close as ended.
19539            let mut rig = rig().await;
19540            let mut child = session(1);
19541            child.scope_ref = "child".to_string();
19542            child.kind = ScopeKind::Worker;
19543            child.parent = Some(ScopeParent::new(
19544                Principal::Reserved {
19545                    module_id: OWNER.to_string(),
19546                },
19547                "s".to_string(),
19548                1,
19549            ));
19550            rig.sync(vec![session(1), child.clone()]).await;
19551            let mut child_route = rig
19552                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19553                .await;
19554            assert_eq!(
19555                child_route.stamp().unwrap().parent_state,
19556                Some(ParentState::Linked)
19557            );
19558            rig.sync(vec![child]).await;
19559            assert!(!rig.live(&child_route));
19560            assert_eq!(
19561                child_route.closed_reason(),
19562                RouteCloseReason::ScopeParentEnded
19563            );
19564        }
19565
19566        #[tokio::test]
19567        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19568        ) {
19569            let mut rig = rig().await;
19570            rig.sync(vec![session(1)]).await;
19571            let mut route = rig
19572                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19573                .await;
19574            let before = rig.forwarding.published_scope_tag(OWNER, "s");
19575            rig.sync(vec![session(1)]).await;
19576            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19577            assert!(rig.live(&route) && route.untouched());
19578
19579            // A call in flight on the route when another carrier is added. A
19580            // forwarded REQUEST holds one credit on the route's flow until the
19581            // module answers; the router takes it exactly like this.
19582            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19583                .forwarding
19584                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19585                .unwrap()
19586            else {
19587                panic!("the route is bound");
19588            };
19589            binding.flow.acquire_tagged(9, false).await.unwrap();
19590            let mut widened = session(1);
19591            widened.carriers.push(carrier(MAGIC, None));
19592            rig.sync(vec![widened]).await;
19593            assert!(rig.live(&route) && route.untouched());
19594            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19595            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19596            // The call's credit is still held on an open flow, so its answer
19597            // will be delivered: closing the route would have closed the flow.
19598            assert_eq!(binding.flow.in_flight(), 1);
19599            binding
19600                .flow
19601                .acquire_tagged(10, false)
19602                .await
19603                .expect("the flow is still open");
19604        }
19605
19606        /// A swap's superseded endpoint keeps its routes until drained; ending
19607        /// the scope closes them there too.
19608        #[tokio::test]
19609        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19610            let mut rig = rig().await;
19611            rig.sync(vec![session(1)]).await;
19612            let mut on_incumbent = rig
19613                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19614                .await;
19615
19616            // Swap plexus: register a candidate and cut over, leaving the
19617            // incumbent superseded with the route still on it.
19618            let (candidate, _candidate_rx) = wide_ctx(9);
19619            let registration = rig
19620                .handler
19621                .registry
19622                .register_candidate_with_control_ops(
19623                    manifest(PLEXUS, PROTOCOL_VERSION),
19624                    PROTOCOL_VERSION,
19625                    candidate.connection_id,
19626                    module_baseline_control_ops(),
19627                )
19628                .unwrap();
19629            rig.forwarding
19630                .register_candidate_module_connection(
19631                    candidate.connection_id,
19632                    PLEXUS.to_string(),
19633                    PROTOCOL_VERSION,
19634                    manifest_concurrency(&registration.manifest),
19635                    candidate.egress.clone(),
19636                )
19637                .unwrap();
19638            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19639            rig.handler
19640                .registry
19641                .promote_candidate(PLEXUS)
19642                .unwrap()
19643                .unwrap();
19644            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19645
19646            rig.sync(Vec::new()).await;
19647            assert!(!rig.live(&on_incumbent));
19648            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19649            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19650            let goodbye = incumbent_rx
19651                .try_recv()
19652                .expect("the superseded endpoint is told")
19653                .frame;
19654            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19655        }
19656    }
19657}
19658
19659#[cfg(test)]
19660mod concurrency_default_exposure_tests {
19661    use super::*;
19662
19663    fn hello_body(role_json: &str) -> Vec<u8> {
19664        format!(
19665            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19666        )
19667        .into_bytes()
19668    }
19669
19670    fn manifest_from(body: &[u8]) -> ModuleManifest {
19671        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19672        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19673            .expect("manifest parses")
19674    }
19675
19676    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19677
19678    #[test]
19679    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19680        let body = hello_body(&format!(
19681            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19682        ));
19683        let manifest = manifest_from(&body);
19684        // Precondition: serde really resolved it to the default, so the typed
19685        // manifest alone cannot answer the question this probe exists for.
19686        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19687        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19688    }
19689
19690    #[test]
19691    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19692        let body = hello_body(&format!(
19693            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19694        ));
19695        let manifest = manifest_from(&body);
19696        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19697        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19698    }
19699
19700    #[test]
19701    fn non_management_roles_are_never_reported() {
19702        let body = hello_body(
19703            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19704        );
19705        let manifest = manifest_from(&body);
19706        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19707    }
19708}